diff options
| author | Dan Willemsen <dwillemsen@google.com> | 2023-08-23 23:12:47 +0000 |
|---|---|---|
| committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2023-08-23 23:12:47 +0000 |
| commit | 54f25be124181b5b5e37d686ae78a2ef43f77594 (patch) | |
| tree | a199f13004f9dca173b6bc46161736ffc224f6fc | |
| parent | 0d8c94d2775dd2629be4247df9847de23c073490 (diff) | |
| parent | eba01819262de24db698eeb259f488ba2b1e89e6 (diff) | |
| download | pybind11-main.tar.gz | |
Upgrade pybind11 to v2.11.0 am: df375684ae am: c3651ffda2 am: b8f469b116 am: ce45d68f7a am: eba0181926HEADmastermain
Original change: https://android-review.googlesource.com/c/platform/external/python/pybind11/+/2722853
Change-Id: Iac83f632be76c6ead72145c9705e5af0233309ac
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
238 files changed, 25644 insertions, 10994 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 85445d41..360760ac 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,6 +1,6 @@ version: 1.0.{build} image: -- Visual Studio 2015 +- Visual Studio 2017 test: off skip_branch_with_pr: true build: @@ -11,11 +11,9 @@ environment: matrix: - PYTHON: 36 CONFIG: Debug - - PYTHON: 27 - CONFIG: Debug install: - ps: | - $env:CMAKE_GENERATOR = "Visual Studio 14 2015" + $env:CMAKE_GENERATOR = "Visual Studio 15 2017" if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" python -W ignore -m pip install --upgrade pip wheel diff --git a/.clang-format b/.clang-format index 8700fca8..b477a160 100644 --- a/.clang-format +++ b/.clang-format @@ -3,19 +3,36 @@ # clang-format --style=llvm --dump-config BasedOnStyle: LLVM AccessModifierOffset: -4 -AlignConsecutiveAssignments: true +AllowShortLambdasOnASingleLine: true AlwaysBreakTemplateDeclarations: Yes BinPackArguments: false BinPackParameters: false BreakBeforeBinaryOperators: All BreakConstructorInitializers: BeforeColon ColumnLimit: 99 +CommentPragmas: 'NOLINT:.*|^ IWYU pragma:' +IncludeBlocks: Regroup IndentCaseLabels: true IndentPPDirectives: AfterHash IndentWidth: 4 Language: Cpp SpaceAfterCStyleCast: true -# SpaceInEmptyBlock: true # too new Standard: Cpp11 +StatementMacros: ['PyObject_HEAD'] TabWidth: 4 +IncludeCategories: + - Regex: '<pybind11/.*' + Priority: -1 + - Regex: 'pybind11.h"$' + Priority: 1 + - Regex: '^".*/?detail/' + Priority: 1 + SortPriority: 2 + - Regex: '^"' + Priority: 1 + SortPriority: 3 + - Regex: '<[[:alnum:]._]+>' + Priority: 4 + - Regex: '.*' + Priority: 5 ... diff --git a/.clang-tidy b/.clang-tidy index e29d9298..23018386 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,13 +1,77 @@ FormatStyle: file -Checks: ' -llvm-namespace-comment, -modernize-use-override, -readability-container-size-empty, -modernize-use-using, -modernize-use-equals-default, -modernize-use-auto, -modernize-use-emplace, -' +Checks: | + *bugprone*, + *performance*, + clang-analyzer-optin.cplusplus.VirtualCall, + clang-analyzer-optin.performance.Padding, + cppcoreguidelines-init-variables, + cppcoreguidelines-prefer-member-initializer, + cppcoreguidelines-pro-type-static-cast-downcast, + cppcoreguidelines-slicing, + google-explicit-constructor, + llvm-namespace-comment, + misc-definitions-in-headers, + misc-misplaced-const, + misc-non-copyable-objects, + misc-static-assert, + misc-throw-by-value-catch-by-reference, + misc-uniqueptr-reset-release, + misc-unused-parameters, + modernize-avoid-bind, + modernize-loop-convert, + modernize-make-shared, + modernize-redundant-void-arg, + modernize-replace-auto-ptr, + modernize-replace-disallow-copy-and-assign-macro, + modernize-replace-random-shuffle, + modernize-shrink-to-fit, + modernize-use-auto, + modernize-use-bool-literals, + modernize-use-default-member-init, + modernize-use-emplace, + modernize-use-equals-default, + modernize-use-equals-delete, + modernize-use-noexcept, + modernize-use-nullptr, + modernize-use-override, + modernize-use-using, + readability-avoid-const-params-in-decls, + readability-braces-around-statements, + readability-const-return-type, + readability-container-size-empty, + readability-delete-null-pointer, + readability-else-after-return, + readability-implicit-bool-conversion, + readability-inconsistent-declaration-parameter-name, + readability-make-member-function-const, + readability-misplaced-array-index, + readability-non-const-parameter, + readability-qualified-auto, + readability-redundant-function-ptr-dereference, + readability-redundant-smartptr-get, + readability-redundant-string-cstr, + readability-simplify-subscript-expr, + readability-static-accessed-through-instance, + readability-static-definition-in-anonymous-namespace, + readability-string-compare, + readability-suspicious-call-argument, + readability-uniqueptr-delete-release, + -bugprone-easily-swappable-parameters, + -bugprone-exception-escape, + -bugprone-reserved-identifier, + -bugprone-unused-raii, + +CheckOptions: +- key: modernize-use-equals-default.IgnoreMacros + value: false +- key: performance-for-range-copy.WarnOnAllAutoCopies + value: true +- key: performance-inefficient-string-concatenation.StrictMode + value: true +- key: performance-unnecessary-value-param.AllowedTypes + value: 'exception_ptr$;' +- key: readability-implicit-bool-conversion.AllowPointerConditions + value: true HeaderFilterRegex: 'pybind11/.*h' diff --git a/.codespell-ignore-lines b/.codespell-ignore-lines new file mode 100644 index 00000000..2a01d63e --- /dev/null +++ b/.codespell-ignore-lines @@ -0,0 +1,24 @@ +template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> + template <typename ThisT> + auto &this_ = static_cast<ThisT &>(*this); + if (load_impl<ThisT>(temp, false)) { + ssize_t nd = 0; + auto trivial = broadcast(buffers, nd, shape); + auto ndim = (size_t) nd; + int nd; + ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; } + using op = op_impl<id, ot, Base, L_type, R_type>; +template <op_id id, op_type ot, typename L, typename R> + template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra> + class_ &def(const detail::op_<id, ot, L, R> &op, const Extra &...extra) { + class_ &def_cast(const detail::op_<id, ot, L, R> &op, const Extra &...extra) { +@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) +struct IntStruct { + explicit IntStruct(int v) : value(v){}; + ~IntStruct() { value = -value; } + IntStruct(const IntStruct &) = default; + IntStruct &operator=(const IntStruct &) = default; + py::class_<IntStruct>(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); })); + py::implicitly_convertible<int, IntStruct>(); + m.def("test", [](int expected, const IntStruct &in) { + [](int expected, const IntStruct &in) { diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..d611e149 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +docs/*.svg binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..4e2c6690 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,9 @@ +*.cmake @henryiii +CMakeLists.txt @henryiii +*.yml @henryiii +*.yaml @henryiii +/tools/ @henryiii +/pybind11/ @henryiii +noxfile.py @henryiii +.clang-format @henryiii +.clang-tidy @henryiii diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 08d9e7cb..ad797439 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -53,6 +53,33 @@ derivative works thereof, in binary and source code form. ## Development of pybind11 +### Quick setup + +To setup a quick development environment, use [`nox`](https://nox.thea.codes). +This will allow you to do some common tasks with minimal setup effort, but will +take more time to run and be less flexible than a full development environment. +If you use [`pipx run nox`](https://pipx.pypa.io), you don't even need to +install `nox`. Examples: + +```bash +# List all available sessions +nox -l + +# Run linters +nox -s lint + +# Run tests on Python 3.9 +nox -s tests-3.9 + +# Build and preview docs +nox -s docs -- serve + +# Build SDists and wheels +nox -s build +``` + +### Full setup + To setup an ideal development environment, run the following commands on a system with CMake 3.14+: @@ -66,11 +93,10 @@ cmake --build build -j4 Tips: -* You can use `virtualenv` (from PyPI) instead of `venv` (which is Python 3 - only). +* You can use `virtualenv` (faster, from PyPI) instead of `venv`. * You can select any name for your environment folder; if it contains "env" it will be ignored by git. -* If you don’t have CMake 3.14+, just add “cmake” to the pip install command. +* If you don't have CMake 3.14+, just add "cmake" to the pip install command. * You can use `-DPYBIND11_FINDPYTHON=ON` to use FindPython on CMake 3.12+ * In classic mode, you may need to set `-DPYTHON_EXECUTABLE=/path/to/python`. FindPython uses `-DPython_ROOT_DIR=/path/to` or @@ -78,7 +104,7 @@ Tips: ### Configuration options -In CMake, configuration options are given with “-D”. Options are stored in the +In CMake, configuration options are given with "-D". Options are stored in the build directory, in the `CMakeCache.txt` file, so they are remembered for each build directory. Two selections are special - the generator, given with `-G`, and the compiler, which is selected based on environment variables `CXX` and @@ -88,12 +114,12 @@ after the initial run. The valid options are: * `-DCMAKE_BUILD_TYPE`: Release, Debug, MinSizeRel, RelWithDebInfo -* `-DPYBIND11_FINDPYTHON=ON`: Use CMake 3.12+’s FindPython instead of the +* `-DPYBIND11_FINDPYTHON=ON`: Use CMake 3.12+'s FindPython instead of the classic, deprecated, custom FindPythonLibs * `-DPYBIND11_NOPYTHON=ON`: Disable all Python searching (disables tests) * `-DBUILD_TESTING=ON`: Enable the tests * `-DDOWNLOAD_CATCH=ON`: Download catch to build the C++ tests -* `-DOWNLOAD_EIGEN=ON`: Download Eigen for the NumPy tests +* `-DDOWNLOAD_EIGEN=ON`: Download Eigen for the NumPy tests * `-DPYBIND11_INSTALL=ON/OFF`: Enable the install target (on by default for the master project) * `-DUSE_PYTHON_INSTALL_DIR=ON`: Try to install into the python dir @@ -132,8 +158,9 @@ tests with these targets: * `test_cmake_build`: Install / subdirectory tests If you want to build just a subset of tests, use -`-DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp"`. If this is -empty, all tests will be built. +`-DPYBIND11_TEST_OVERRIDE="test_callbacks;test_pickling"`. If this is +empty, all tests will be built. Tests are specified without an extension if they need both a .py and +.cpp file. You may also pass flags to the `pytest` target by editing `tests/pytest.ini` or by using the `PYTEST_ADDOPTS` environment variable @@ -203,16 +230,19 @@ of the pybind11 repo. [`clang-tidy`][clang-tidy] performs deeper static code analyses and is more complex to run, compared to `clang-format`, but support for `clang-tidy` is built into the pybind11 CMake configuration. To run `clang-tidy`, the -following recipe should work. Files will be modified in place, so you can -use git to monitor the changes. +following recipe should work. Run the `docker` command from the top-level +directory inside your pybind11 git clone. Files will be modified in place, +so you can use git to monitor the changes. ```bash -docker run --rm -v $PWD:/pybind11 -it silkeh/clang:10 -apt-get update && apt-get install python3-dev python3-pytest -cmake -S pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);-fix" -cmake --build build +docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:15-bullseye +apt-get update && apt-get install -y git python3-dev python3-pytest +cmake -S /mounted_pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color" -DDOWNLOAD_EIGEN=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=17 +cmake --build build -j 2 ``` +You can add `--fix` to the options list if you want. + ### Include what you use To run include what you use, install (`brew install include-what-you-use` on @@ -228,7 +258,7 @@ The report is sent to stderr; you can pipe it into a file if you wish. ### Build recipes This builds with the Intel compiler (assuming it is in your path, along with a -recent CMake and Python 3): +recent CMake and Python): ```bash python3 -m venv venv diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md deleted file mode 100644 index ae36ea65..00000000 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: Bug Report -about: File an issue about a bug -title: "[BUG] " ---- - - -Make sure you've completed the following steps before submitting your issue -- thank you! - -1. Make sure you've read the [documentation][]. Your issue may be addressed there. -2. Search the [issue tracker][] to verify that this hasn't already been reported. +1 or comment there if it has. -3. Consider asking first in the [Gitter chat room][]. -4. Include a self-contained and minimal piece of code that reproduces the problem. If that's not possible, try to make the description as clear as possible. - a. If possible, make a PR with a new, failing test to give us a starting point to work on! - -[documentation]: https://pybind11.readthedocs.io -[issue tracker]: https://github.com/pybind/pybind11/issues -[Gitter chat room]: https://gitter.im/pybind/Lobby - -*After reading, remove this checklist and the template text in parentheses below.* - -## Issue description - -(Provide a short description, state the expected behavior and what actually happens.) - -## Reproducible example code - -(The code should be minimal, have no external dependencies, isolate the function(s) that cause breakage. Submit matched and complete C++ and Python snippets that can be easily compiled and run to diagnose the issue.) diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..4f1e78f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,61 @@ +name: Bug Report +description: File an issue about a bug +title: "[BUG]: " +labels: [triage] +body: + - type: markdown + attributes: + value: | + Please do your best to make the issue as easy to act on as possible, and only submit here if there is clearly a problem with pybind11 (ask first if unsure). **Note that a reproducer in a PR is much more likely to get immediate attention.** + + - type: checkboxes + id: steps + attributes: + label: Required prerequisites + description: Make sure you've completed the following steps before submitting your issue -- thank you! + options: + - label: Make sure you've read the [documentation](https://pybind11.readthedocs.io). Your issue may be addressed there. + required: true + - label: Search the [issue tracker](https://github.com/pybind/pybind11/issues) and [Discussions](https:/pybind/pybind11/discussions) to verify that this hasn't already been reported. +1 or comment there if it has. + required: true + - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new). + required: false + + - type: input + id: version + attributes: + label: What version (or hash if on master) of pybind11 are you using? + validations: + required: true + + - type: textarea + id: description + attributes: + label: Problem description + placeholder: >- + Provide a short description, state the expected behavior and what + actually happens. Include relevant information like what version of + pybind11 you are using, what system you are on, and any useful commands + / output. + validations: + required: true + + - type: textarea + id: code + attributes: + label: Reproducible example code + placeholder: >- + The code should be minimal, have no external dependencies, isolate the + function(s) that cause breakage. Submit matched and complete C++ and + Python snippets that can be easily compiled and run to diagnose the + issue. — Note that a reproducer in a PR is much more likely to get + immediate attention: failing tests in the pybind11 CI are the best + starting point for working out fixes. + render: text + + - type: input + id: regression + attributes: + label: Is this a regression? Put the last known working version here if it is. + description: Put the last known working version here if this is a regression. + value: Not a regression diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 20e74313..27f9a804 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,8 @@ blank_issues_enabled: false contact_links: + - name: Ask a question + url: https://github.com/pybind/pybind11/discussions/new + about: Please ask and answer questions here, or propose new ideas. - name: Gitter room url: https://gitter.im/pybind/Lobby about: A room for discussing pybind11 with an active community diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md deleted file mode 100644 index 5f6ec81e..00000000 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Feature Request -about: File an issue about adding a feature -title: "[FEAT] " ---- - - -Make sure you've completed the following steps before submitting your issue -- thank you! - -1. Check if your feature has already been mentioned / rejected / planned in other issues. -2. If those resources didn't help, consider asking in the [Gitter chat room][] to see if this is interesting / useful to a larger audience and possible to implement reasonably, -4. If you have a useful feature that passes the previous items (or not suitable for chat), please fill in the details below. - -[Gitter chat room]: https://gitter.im/pybind/Lobby - -*After reading, remove this checklist.* diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md deleted file mode 100644 index b199b6ee..00000000 --- a/.github/ISSUE_TEMPLATE/question.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: Question -about: File an issue about unexplained behavior -title: "[QUESTION] " ---- - -If you have a question, please check the following first: - -1. Check if your question has already been answered in the [FAQ][] section. -2. Make sure you've read the [documentation][]. Your issue may be addressed there. -3. If those resources didn't help and you only have a short question (not a bug report), consider asking in the [Gitter chat room][] -4. Search the [issue tracker][], including the closed issues, to see if your question has already been asked/answered. +1 or comment if it has been asked but has no answer. -5. If you have a more complex question which is not answered in the previous items (or not suitable for chat), please fill in the details below. -6. Include a self-contained and minimal piece of code that illustrates your question. If that's not possible, try to make the description as clear as possible. - -[FAQ]: http://pybind11.readthedocs.io/en/latest/faq.html -[documentation]: https://pybind11.readthedocs.io -[issue tracker]: https://github.com/pybind/pybind11/issues -[Gitter chat room]: https://gitter.im/pybind/Lobby - -*After reading, remove this checklist.* diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 73273365..2c7d1708 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,12 +5,3 @@ updates: directory: "/" schedule: interval: "daily" - ignore: - # Official actions have moving tags like v1 - # that are used, so they don't need updates here - - dependency-name: "actions/checkout" - - dependency-name: "actions/setup-python" - - dependency-name: "actions/cache" - - dependency-name: "actions/upload-artifact" - - dependency-name: "actions/download-artifact" - - dependency-name: "actions/labeler" diff --git a/.github/matchers/pylint.json b/.github/matchers/pylint.json new file mode 100644 index 00000000..e3a6bd16 --- /dev/null +++ b/.github/matchers/pylint.json @@ -0,0 +1,32 @@ +{ + "problemMatcher": [ + { + "severity": "warning", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): ([A-DF-Z]\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-warning" + }, + { + "severity": "error", + "pattern": [ + { + "regexp": "^([^:]+):(\\d+):(\\d+): (E\\d+): \\033\\[[\\d;]+m([^\\033]+).*$", + "file": 1, + "line": 2, + "column": 3, + "code": 4, + "message": 5 + } + ], + "owner": "pylint-error" + } + ] +} diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 97a6ff7d..54b7f510 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,7 @@ +<!-- +Title (above): please place [branch_name] at the beginning if you are targeting a branch other than master. *Do not target stable*. +It is recommended to use conventional commit format, see conventionalcommits.org, but not required. +--> ## Description <!-- Include relevant issues or PRs here, describe what changed and why --> diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f90c1995..48f7c5e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,20 @@ on: - stable - v* +permissions: read-all + +concurrency: + group: test-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + PIP_ONLY_BINARY: numpy + FORCE_COLOR: 3 + PYTEST_TIMEOUT: 300 + # For cmake: + VERBOSE: 1 + jobs: # This is the "main" test suite, which tests a large number of different # versions of default compilers and Python versions in GitHub Actions. @@ -16,66 +30,68 @@ jobs: strategy: fail-fast: false matrix: - runs-on: [ubuntu-latest, windows-latest, macos-latest] + runs-on: [ubuntu-20.04, windows-2022, macos-latest] python: - - 2.7 - - 3.5 - - 3.6 - - 3.9 - # - 3.10-dev # Re-enable once 3.10.0a5 is released - - pypy2 - - pypy3 + - '3.6' + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - 'pypy-3.8' + - 'pypy-3.9' + - 'pypy-3.10' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the # existing keys match. # - # We support three optional keys: args (both build), args1 (first - # build), and args2 (second build). + # We support an optional key: args, for cmake args include: # Just add a key - - runs-on: ubuntu-latest - python: 3.6 + - runs-on: ubuntu-20.04 + python: '3.6' args: > -DPYBIND11_FINDPYTHON=ON - - runs-on: windows-latest - python: 3.6 + -DCMAKE_CXX_FLAGS="-D_=1" + - runs-on: ubuntu-20.04 + python: 'pypy-3.8' args: > -DPYBIND11_FINDPYTHON=ON - - # These items will be removed from the build matrix, keys must match. - exclude: - # Currently 32bit only, and we build 64bit - - runs-on: windows-latest - python: pypy2 - - runs-on: windows-latest - python: pypy3 - - # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine. - - runs-on: ubuntu-latest - python: pypy2 + - runs-on: windows-2019 + python: '3.6' + args: > + -DPYBIND11_FINDPYTHON=ON + # Inject a couple Windows 2019 runs + - runs-on: windows-2019 + python: '3.9' name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}" runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - - name: Setup Boost (Windows / Linux latest) - shell: bash - run: echo "BOOST_ROOT=$BOOST_ROOT_1_72_0" >> $GITHUB_ENV + - name: Setup Boost (Linux) + # Can't use boost + define _ + if: runner.os == 'Linux' && matrix.python != '3.6' + run: sudo apt-get install libboost-dev + + - name: Setup Boost (macOS) + if: runner.os == 'macOS' + run: brew install boost - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.7 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Cache wheels if: runner.os == 'macOS' - uses: actions/cache@v2 + uses: actions/cache@v3 with: # This path is specific to macOS - we really only need it for PyPy NumPy wheels # See https://github.com/actions/cache/blob/master/examples.md#python---pip @@ -85,17 +101,20 @@ jobs: key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }} - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt - name: Setup annotations on Linux if: runner.os == 'Linux' run: python -m pip install pytest-github-actions-annotate-failures # First build - C++11 mode and inplace + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here. - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 @@ -109,7 +128,7 @@ jobs: - name: C++11 tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 @@ -119,15 +138,16 @@ jobs: run: git clean -fdx # Second build - C++17 mode and in a build directory - - name: Configure ${{ matrix.args2 }} + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. + - name: Configure C++17 run: > cmake -S . -B build2 -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 ${{ matrix.args }} - ${{ matrix.args2 }} - name: Build run: cmake --build build2 -j 2 @@ -137,32 +157,36 @@ jobs: - name: C++ tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build build2 --target cpptest - - name: Interface test - run: cmake --build build2 --target test_cmake_build + # Third build - C++17 mode with unstable ABI + - name: Configure (unstable ABI) + run: > + cmake -S . -B build3 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + -DPYBIND11_INTERNALS_VERSION=10000000 + ${{ matrix.args }} - # Eventually Microsoft might have an action for setting up - # MSVC, but for now, this action works: - - name: Prepare compiler environment for Windows 🐍 2.7 - if: matrix.python == 2.7 && runner.os == 'Windows' - uses: ilammy/msvc-dev-cmd@v1 - with: - arch: x64 + - name: Build (unstable ABI) + run: cmake --build build3 -j 2 - # This makes two environment variables available in the following step(s) - - name: Set Windows 🐍 2.7 environment variables - if: matrix.python == 2.7 && runner.os == 'Windows' - shell: bash - run: | - echo "DISTUTILS_USE_SDK=1" >> $GITHUB_ENV - echo "MSSdk=1" >> $GITHUB_ENV + - name: Python tests (unstable ABI) + run: cmake --build build3 --target pytest + + - name: Interface test + run: cmake --build build2 --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools + if: "!(matrix.runs-on == 'windows-2022')" deadsnakes: @@ -170,30 +194,31 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.9 + # TODO: Fails on 3.10, investigate + - python-version: "3.9" python-debug: true valgrind: true - - python-version: 3.10-dev + - python-version: "3.11" python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v2.1.1 + uses: deadsnakes/action@v3.0.1 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.7 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Valgrind cache if: matrix.valgrind - uses: actions/cache@v2 + uses: actions/cache@v3 id: cache-valgrind with: path: valgrind @@ -218,7 +243,8 @@ jobs: sudo apt-get install libc6-dbg # Needed by Valgrind - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt - name: Configure run: > @@ -258,19 +284,30 @@ jobs: - dev std: - 11 + container_suffix: + - "" include: - clang: 5 std: 14 - clang: 10 - std: 20 - - clang: 10 std: 17 + - clang: 11 + std: 20 + - clang: 12 + std: 20 + - clang: 13 + std: 20 + - clang: 14 + std: 20 + - clang: 15 + std: 20 + container_suffix: "-bullseye" name: "🐍 3 • Clang ${{ matrix.clang }} • C++${{ matrix.std }} • x64" - container: "silkeh/clang:${{ matrix.clang }}" + container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Add wget and python3 run: apt-get update && apt-get install -y python3-dev python3-numpy python3-pytest libeigen3-dev @@ -300,11 +337,11 @@ jobs: # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest - name: "🐍 3.8 • CUDA 11 • Ubuntu 20.04" - container: nvidia/cuda:11.0-devel-ubuntu20.04 + name: "🐍 3.10 • CUDA 12.2 • Ubuntu 22.04" + container: nvidia/cuda:12.2.0-devel-ubuntu22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 # tzdata will try to ask for the timezone, so set the DEBIAN_FRONTEND - name: Install 🐍 3 @@ -328,7 +365,7 @@ jobs: # container: centos:8 # # steps: -# - uses: actions/checkout@v2 +# - uses: actions/checkout@v3 # # - name: Add Python 3 and a few requirements # run: yum update -y && yum install -y git python3-devel python3-numpy python3-pytest make environment-modules @@ -366,33 +403,34 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: + if: ${{ false }} # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690 runs-on: ubuntu-latest - name: "🐍 3 • CentOS7 / PGI 20.9 • x64" + name: "🐍 3 • CentOS7 / PGI 22.9 • x64" container: centos:7 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Add Python 3 and a few requirements - run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 + run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils - name: Install NVidia HPC SDK - run: yum -y install https://developer.download.nvidia.com/hpc-sdk/20.9/nvhpc-20-9-20.9-1.x86_64.rpm https://developer.download.nvidia.com/hpc-sdk/20.9/nvhpc-2020-20.9-1.x86_64.rpm + run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 # On CentOS 7, we have to filter a few tests (compiler internal error) - # and allow deeper templete recursion (not needed on CentOS 8 with a newer + # and allow deeper template recursion (not needed on CentOS 8 with a newer # standard library). On some systems, you many need further workarounds: # https://github.com/pybind/pybind11/pull/2475 - name: Configure shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/20.9 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ -DCMAKE_CXX_STANDARD=11 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ - -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" # Building before installing Pip should produce a warning but not an error - name: Build @@ -419,20 +457,20 @@ jobs: strategy: fail-fast: false matrix: - gcc: - - 7 - - latest - std: - - 11 include: - - gcc: 10 - std: 20 + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } name: "🐍 3 • GCC ${{ matrix.gcc }} • C++${{ matrix.std }}• x64" container: "gcc:${{ matrix.gcc }}" steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Add Python 3 run: apt-get update; apt-get install -y python3-dev python3-numpy python3-pytest python3-pip libeigen3-dev @@ -441,7 +479,7 @@ jobs: run: python3 -m pip install --upgrade pip - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.7 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure shell: bash @@ -464,6 +502,24 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + - name: Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + shell: bash + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial --target pytest # Testing on ICC using the oneAPI apt repo icc: @@ -474,7 +530,7 @@ jobs: name: "🐍 3 • ICC latest • x64" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Add apt repo run: | @@ -495,7 +551,7 @@ jobs: - name: Install dependencies run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e - python3 -m pip install -r tests/requirements.txt --prefer-binary + python3 -m pip install -r tests/requirements.txt - name: Configure C++11 run: | @@ -569,29 +625,37 @@ jobs: strategy: fail-fast: false matrix: - centos: - - 7 # GCC 4.8 - - 8 + container: + - "centos:7" # GCC 4.8 + - "almalinux:8" + - "almalinux:9" - name: "🐍 3 • CentOS ${{ matrix.centos }} • x64" - container: "centos:${{ matrix.centos }}" + name: "🐍 3 • ${{ matrix.container }} • x64" + container: "${{ matrix.container }}" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Add Python 3 + - name: Add Python 3 (RHEL 7) + if: matrix.container == 'centos:7' run: yum update -y && yum install -y python3-devel gcc-c++ make git + - name: Add Python 3 (RHEL 8+) + if: matrix.container != 'centos:7' + run: dnf update -y && dnf install -y python3-devel gcc-c++ make git + - name: Update pip run: python3 -m pip install --upgrade pip - name: Install dependencies - run: python3 -m pip install cmake -r tests/requirements.txt --prefer-binary + run: | + python3 -m pip install cmake -r tests/requirements.txt - name: Configure shell: bash run: > cmake -S . -B build + -DCMAKE_BUILD_TYPE=MinSizeRel -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON @@ -613,18 +677,18 @@ jobs: # This tests an "install" with the CMake tools install-classic: - name: "🐍 3.5 • Debian • x86 • Install" + name: "🐍 3.7 • Debian • x86 • Install" runs-on: ubuntu-latest - container: i386/debian:stretch + container: i386/debian:buster steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v1 # Required to run inside docker - name: Install requirements run: | apt-get update apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip - pip3 install "pytest==3.1.*" + pip3 install "pytest==6.*" - name: Configure for install run: > @@ -649,33 +713,32 @@ jobs: -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") working-directory: /build-tests - - name: Run tests + - name: Python tests run: make pytest -j 2 working-directory: /build-tests # This verifies that the documentation is not horribly broken, and does a - # basic sanity check on the SDist. + # basic validation check on the SDist. doxygen: name: "Documentation build test" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" - name: Install Doxygen run: sudo apt-get install -y doxygen librsvg2-bin # Changed to rsvg-convert in 20.04 - - name: Install docs & setup requirements - run: python3 -m pip install -r docs/requirements.txt - - name: Build docs - run: python3 -m sphinx -W -b html docs docs/.build + run: pipx run nox -s docs - name: Make SDist - run: python3 setup.py sdist + run: pipx run nox -s build -- --sdist - run: git status --ignored @@ -687,7 +750,7 @@ jobs: - name: Compare Dists (headers only) working-directory: include run: | - python3 -m pip install --user -U ../dist/* + python3 -m pip install --user -U ../dist/*.tar.gz installed=$(python3 -c "import pybind11; print(pybind11.get_include() + '/pybind11')") diff -rq $installed ./pybind11 @@ -696,42 +759,43 @@ jobs: fail-fast: false matrix: python: - - 3.5 - 3.6 - 3.7 - 3.8 - 3.9 - - pypy3 - # TODO: fix hang on pypy2 include: - python: 3.9 - args: -DCMAKE_CXX_STANDARD=20 -DDOWNLOAD_EIGEN=OFF + args: -DCMAKE_CXX_STANDARD=20 - python: 3.8 args: -DCMAKE_CXX_STANDARD=17 + - python: 3.7 + args: -DCMAKE_CXX_STANDARD=14 + name: "🐍 ${{ matrix.python }} • MSVC 2019 • x86 ${{ matrix.args }}" - runs-on: windows-latest + runs-on: windows-2019 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.7 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt # First build - C++11 mode and inplace - name: Configure ${{ matrix.args }} @@ -745,102 +809,357 @@ jobs: - name: Build C++11 run: cmake --build build -j 2 - - name: Run tests + - name: Python tests run: cmake --build build -t pytest - win32-msvc2015: - name: "🐍 ${{ matrix.python }} • MSVC 2015 • x64" - runs-on: windows-latest + win32-debug: strategy: fail-fast: false matrix: python: - - 2.7 - - 3.6 - - 3.7 - # todo: check/cpptest does not support 3.8+ yet + - 3.8 + - 3.9 + + include: + - python: 3.9 + args: -DCMAKE_CXX_STANDARD=20 + - python: 3.8 + args: -DCMAKE_CXX_STANDARD=17 + + name: "🐍 ${{ matrix.python }} • MSVC 2019 (Debug) • x86 ${{ matrix.args }}" + runs-on: windows-2019 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Setup 🐍 ${{ matrix.python }} - uses: actions/setup-python@v2 + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.7 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: - toolset: 14.0 + arch: x86 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt # First build - C++11 mode and inplace - - name: Configure + - name: Configure ${{ matrix.args }} run: > cmake -S . -B build - -G "Visual Studio 14 2015" -A x64 + -G "Visual Studio 16 2019" -A Win32 + -DCMAKE_BUILD_TYPE=Debug -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON + ${{ matrix.args }} + - name: Build C++11 + run: cmake --build build --config Debug -j 2 - - name: Build C++14 - run: cmake --build build -j 2 - - - name: Run all checks - run: cmake --build build -t check + - name: Python tests + run: cmake --build build --config Debug -t pytest - win32-msvc2017: - name: "🐍 ${{ matrix.python }} • MSVC 2017 • x64" - runs-on: windows-2016 + windows-2022: strategy: fail-fast: false matrix: python: - - 2.7 - - 3.5 - - 3.7 - std: - - 14 + - 3.9 - include: - - python: 2.7 - std: 17 - args: > - -DCMAKE_CXX_FLAGS="/permissive- /EHsc /GR" + name: "🐍 ${{ matrix.python }} • MSVC 2022 C++20 • x64" + runs-on: windows-2022 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Setup 🐍 ${{ matrix.python }} - uses: actions/setup-python@v2 + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.7 - - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python3 -m pip install -r tests/requirements.txt - # First build - C++11 mode and inplace - - name: Configure + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Configure C++20 run: > cmake -S . -B build - -G "Visual Studio 15 2017" -A x64 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON - -DCMAKE_CXX_STANDARD=${{ matrix.std }} - ${{ matrix.args }} + -DCMAKE_CXX_STANDARD=20 - - name: Build ${{ matrix.std }} + - name: Build C++20 run: cmake --build build -j 2 - - name: Run all checks - run: cmake --build build -t check + - name: Python tests + run: cmake --build build --target pytest + + - name: C++20 tests + run: cmake --build build --target cpptest -j 2 + + - name: Interface test C++20 + run: cmake --build build --target test_cmake_build + + - name: Configure C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=20 + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest + + mingw: + name: "🐍 3 • windows-latest • ${{ matrix.sys }}" + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + strategy: + fail-fast: false + matrix: + include: + - { sys: mingw64, env: x86_64 } + - { sys: mingw32, env: i686 } + steps: + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{matrix.sys}} + install: >- + git + mingw-w64-${{matrix.env}}-gcc + mingw-w64-${{matrix.env}}-python-pip + mingw-w64-${{matrix.env}}-python-numpy + mingw-w64-${{matrix.env}}-python-scipy + mingw-w64-${{matrix.env}}-cmake + mingw-w64-${{matrix.env}}-make + mingw-w64-${{matrix.env}}-python-pytest + mingw-w64-${{matrix.env}}-eigen3 + mingw-w64-${{matrix.env}}-boost + mingw-w64-${{matrix.env}}-catch + + - uses: actions/checkout@v3 + + - name: Configure C++11 + # LTO leads to many undefined reference like + # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&) + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build + + - name: Build C++11 + run: cmake --build build -j 2 + + - name: Python tests C++11 + run: cmake --build build --target pytest -j 2 + + - name: C++11 tests + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target cpptest -j 2 + + - name: Interface test C++11 + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + - name: Configure C++14 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2 + + - name: Build C++14 + run: cmake --build build2 -j 2 + + - name: Python tests C++14 + run: cmake --build build2 --target pytest -j 2 + + - name: C++14 tests + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target cpptest -j 2 + + - name: Interface test C++14 + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build2 --target test_cmake_build + + - name: Clean directory + run: git clean -fdx + + - name: Configure C++17 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3 + + - name: Build C++17 + run: cmake --build build3 -j 2 + + - name: Python tests C++17 + run: cmake --build build3 --target pytest -j 2 + + - name: C++17 tests + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target cpptest -j 2 + + - name: Interface test C++17 + run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target test_cmake_build + + windows_clang: + + strategy: + matrix: + os: [windows-latest] + python: ['3.10'] + + runs-on: "${{ matrix.os }}" + + name: "🐍 ${{ matrix.python }} • ${{ matrix.os }} • clang-latest" + + steps: + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Install ninja-build tool + uses: seanmiddleditch/gha-setup-ninja@v3 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show Clang++ version + run: clang++ --version + + - name: Show CMake version + run: cmake --version + + # TODO: WERROR=ON + - name: Configure Clang + run: > + cmake -G Ninja -S . -B . + -DPYBIND11_WERROR=OFF + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx + + macos_brew_install_llvm: + name: "macos-latest • brew install llvm" + runs-on: macos-latest + + env: + # https://apple.stackexchange.com/questions/227026/how-to-install-recent-clang-with-homebrew + LDFLAGS: '-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib' + + steps: + - name: Update PATH + run: echo "/usr/local/opt/llvm/bin" >> $GITHUB_PATH + + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Show Clang++ version before brew install llvm + run: clang++ --version + + - name: brew install llvm + run: brew install llvm + + - name: Show Clang++ version after brew install llvm + run: clang++ --version + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Run pip installs + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r tests/requirements.txt + python3 -m pip install numpy + python3 -m pip install scipy + + - name: Show CMake version + run: cmake --version + + - name: CMake Configure + run: > + cmake -S . -B . + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: CMake Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest -j 2 + + - name: Clean directory + run: git clean -fdx diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 578dba63..ec7cd612 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -9,6 +9,14 @@ on: - stable - v* +permissions: + contents: read + +env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + # For cmake: + VERBOSE: 1 + jobs: # This tests various versions of CMake in various combinations, to make sure # the configure step passes. @@ -16,35 +24,35 @@ jobs: strategy: fail-fast: false matrix: - runs-on: [ubuntu-latest, macos-latest, windows-latest] + runs-on: [ubuntu-20.04, macos-latest, windows-latest] arch: [x64] - cmake: [3.18] + cmake: ["3.26"] include: - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 arch: x64 - cmake: 3.4 + cmake: "3.5" - - runs-on: macos-latest + - runs-on: ubuntu-20.04 arch: x64 - cmake: 3.7 + cmake: "3.27" - - runs-on: windows-2016 - arch: x86 - cmake: 3.8 + - runs-on: macos-latest + arch: x64 + cmake: "3.7" - - runs-on: windows-2016 - arch: x86 - cmake: 3.18 + - runs-on: windows-2019 + arch: x64 # x86 compilers seem to be missing on 2019 image + cmake: "3.18" name: 🐍 3.7 • CMake ${{ matrix.cmake }} • ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Python 3.7 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.7 architecture: ${{ matrix.arch }} @@ -55,7 +63,7 @@ jobs: # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} - uses: jwlawson/actions-setup-cmake@v1.7 + uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: ${{ matrix.cmake }} diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 5cebed17..b8242ee5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -12,35 +12,49 @@ on: - stable - "v*" +permissions: + contents: read + +env: + FORCE_COLOR: 3 + # For cmake: + VERBOSE: 1 + jobs: pre-commit: name: Format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Add matchers + run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/matchers/pylint.json" + - uses: pre-commit/action@v3.0.0 with: # Slow hooks are marked with manual - slow is okay here, run them too extra_args: --hook-stage manual --all-files clang-tidy: + # When making changes here, please also review the "Clang-Tidy" section + # in .github/CONTRIBUTING.md and update as needed. name: Clang-Tidy runs-on: ubuntu-latest - container: silkeh/clang:10 + container: silkeh/clang:15-bullseye steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install requirements - run: apt-get update && apt-get install -y python3-dev python3-pytest + run: apt-get update && apt-get install -y git python3-dev python3-pytest - name: Configure run: > cmake -S . -B build - -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--warnings-as-errors=*" + -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color;--warnings-as-errors=*" -DDOWNLOAD_EIGEN=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=17 - name: Build - run: cmake --build build -j 2 + run: cmake --build build -j 2 -- --keep-going diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index d2b59796..858a4a0e 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,14 +3,23 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: label: name: Labeler runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: actions/labeler@main - if: github.event.pull_request.merged == true + if: > + github.event.pull_request.merged == true && + !startsWith(github.event.pull_request.title, 'chore(deps):') && + !startsWith(github.event.pull_request.title, 'ci(fix):') && + !startsWith(github.event.pull_request.title, 'docs(changelog):') with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/labeler_merged.yml diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 4414a12e..d6687b44 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -12,24 +12,32 @@ on: types: - published +permissions: + contents: read + +env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + PIP_ONLY_BINARY: numpy + jobs: # This builds the sdists and wheels and makes sure the files are exactly as - # expected. Using Windows and Python 2.7, since that is often the most + # expected. Using Windows and Python 3.6, since that is often the most # challenging matrix element. test-packaging: - name: 🐍 2.7 • 📦 tests • windows-latest + name: 🐍 3.6 • 📦 tests • windows-latest runs-on: windows-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Setup 🐍 2.7 - uses: actions/setup-python@v2 + - name: Setup 🐍 3.6 + uses: actions/setup-python@v4 with: - python-version: 2.7 + python-version: 3.6 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt - name: Python Packaging tests run: pytest tests/extra_python_package/ @@ -42,15 +50,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup 🐍 3.8 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: 3.8 - name: Prepare env - run: python -m pip install -r tests/requirements.txt build twine --prefer-binary + run: | + python -m pip install -r tests/requirements.txt build twine - name: Python Packaging tests run: pytest tests/extra_python_package/ @@ -64,13 +73,13 @@ jobs: run: twine check dist/* - name: Save standard package - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: standard path: dist/pybind11-* - name: Save global package - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: global path: dist/pybind11_global-* @@ -85,19 +94,21 @@ jobs: needs: [packaging] steps: - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 + with: + python-version: "3.x" # Downloads all to directories matching the artifact names - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.4.1 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} - packages_dir: standard/ + packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.4.1 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password_global }} - packages_dir: global/ + packages-dir: global/ diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml new file mode 100644 index 00000000..dd8a1c96 --- /dev/null +++ b/.github/workflows/upstream.yml @@ -0,0 +1,116 @@ +name: Upstream + +on: + workflow_dispatch: + pull_request: + +permissions: + contents: read + +concurrency: + group: upstream-${{ github.ref }} + cancel-in-progress: true + +env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + PIP_ONLY_BINARY: ":all:" + # For cmake: + VERBOSE: 1 + +jobs: + standard: + name: "🐍 3.12 latest • ubuntu-latest • x64" + runs-on: ubuntu-latest + # Only runs when the 'python dev' label is selected + if: "contains(github.event.pull_request.labels.*.name, 'python dev')" + + steps: + - uses: actions/checkout@v3 + + - name: Setup Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: "3.12-dev" + + - name: Setup Boost + run: sudo apt-get install libboost-dev + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show platform info + run: | + python -m platform + cmake --version + pip list + + # First build - C++11 mode and inplace + - name: Configure C++11 + run: > + cmake -S . -B build11 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=11 + -DCMAKE_BUILD_TYPE=Debug + + - name: Build C++11 + run: cmake --build build11 -j 2 + + - name: Python tests C++11 + run: cmake --build build11 --target pytest -j 2 + + - name: C++11 tests + run: cmake --build build11 --target cpptest -j 2 + + - name: Interface test C++11 + run: cmake --build build11 --target test_cmake_build + + # Second build - C++17 mode and in a build directory + - name: Configure C++17 + run: > + cmake -S . -B build17 + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + + - name: Build C++17 + run: cmake --build build17 -j 2 + + - name: Python tests C++17 + run: cmake --build build17 --target pytest + + - name: C++17 tests + run: cmake --build build17 --target cpptest + + # Third build - C++17 mode with unstable ABI + - name: Configure (unstable ABI) + run: > + cmake -S . -B build17max + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=17 + -DPYBIND11_INTERNALS_VERSION=10000000 + + - name: Build (unstable ABI) + run: cmake --build build17max -j 2 + + - name: Python tests (unstable ABI) + run: cmake --build build17max --target pytest + + - name: Interface test (unstable ABI) + run: cmake --build build17max --target test_cmake_build + + # This makes sure the setup_helpers module can build packages using + # setuptools + - name: Setuptools helpers test + run: | + pip install setuptools + pytest tests/extra_setuptools @@ -41,3 +41,6 @@ pybind11Targets.cmake /.vscode /pybind11/include/* /pybind11/share/* +/docs/_build/* +.ipynb_checkpoints/ +tests/main.cpp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6781ac4f..86ac965d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,89 +12,142 @@ # # See https://github.com/pre-commit/pre-commit + +ci: + autoupdate_commit_msg: "chore(deps): update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + autoupdate_schedule: monthly + +# third-party content +exclude: ^tools/JoinPaths.cmake$ + repos: + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v16.0.6" + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: "23.3.0" # Keep in sync with blacken-docs + hooks: + - id: black + +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.276 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.4.1" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: + - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. + - nox + - rich + - types-setuptools + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: "v4.4.0" hooks: - id: check-added-large-files - id: check-case-conflict + - id: check-docstring-first - id: check-merge-conflict - id: check-symlinks + - id: check-toml - id: check-yaml - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending - id: requirements-txt-fixer - id: trailing-whitespace - - id: fix-encoding-pragma -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: 20.8b1 +# Also code format the docs +- repo: https://github.com/asottile/blacken-docs + rev: "1.14.0" hooks: - - id: black - # By default, this ignores pyi files, though black supports them - types: [text] - files: \.pyi?$ + - id: blacken-docs + additional_dependencies: + - black==23.3.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.1.9 + rev: "v1.5.1" hooks: - id: remove-tabs -# Flake8 also supports pre-commit natively (same author) -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 +# Avoid directional quotes +- repo: https://github.com/sirosen/texthooks + rev: "0.5.0" hooks: - - id: flake8 - additional_dependencies: [flake8-bugbear, pep8-naming] - exclude: ^(docs/.*|tools/.*)$ - -# CMake formatting -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: v0.6.13 - hooks: - - id: cmake-format - additional_dependencies: [pyyaml] - types: [file] - files: (\.cmake|CMakeLists.txt)(.in)?$ + - id: fix-ligatures + - id: fix-smartquotes -# Check static types with mypy -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.800 +# Checking for common mistakes +- repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.10.0" hooks: - - id: mypy - # The default Python type ignores .pyi files, so let's rerun if detected - types: [text] - files: ^pybind11.*\.pyi?$ - # Running per-file misbehaves a bit, so just run on all files, it's fast - pass_filenames: false + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.46" + rev: "0.49" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed stages: [manual] additional_dependencies: [cmake, ninja] -# The original pybind11 checks for a few C++ style items +# Check for spelling +# Use tools/codespell_ignore_lines_from_errors.py +# to rebuild .codespell-ignore-lines +- repo: https://github.com/codespell-project/codespell + rev: "v2.2.5" + hooks: + - id: codespell + exclude: ".supp$" + args: ["-x.codespell-ignore-lines", "-Lccompiler"] + +# Check for common shell mistakes +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: "v0.9.0.5" + hooks: + - id: shellcheck + +# Disallow some common capitalization mistakes - repo: local hooks: - id: disallow-caps name: Disallow improper capitalization language: pygrep - entry: PyBind|Numpy|Cmake|CCache - exclude: .pre-commit-config.yaml + entry: PyBind|Numpy|Cmake|CCache|PyTest + exclude: ^\.pre-commit-config.yaml$ -- repo: local +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v3.0.0a6" hooks: - - id: check-style - name: Classic check-style - language: system - types: - - c++ - entry: ./tools/check-style.sh + - id: pylint + files: ^pybind11 diff --git a/CMakeLists.txt b/CMakeLists.txt index ded4dad9..87ec1034 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,15 +5,20 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) +endif() + +# Avoid infinite recursion if tests include this as a subdirectory +if(DEFINED PYBIND11_MASTER_PROJECT) + return() endif() # Extract project version from source @@ -45,13 +50,8 @@ if(NOT pybind11_FIND_QUIETLY) message(STATUS "pybind11 v${pybind11_VERSION} ${pybind11_VERSION_TYPE}") endif() -# Avoid infinite recursion if tests include this as a subdirectory -if(DEFINED PYBIND11_MASTER_PROJECT) - set(PYBIND11_TEST OFF) -endif() - # Check if pybind11 is being used directly or via add_subdirectory -if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR AND NOT DEFINED PYBIND11_MASTER_PROJECT) +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) ### Warn if not an out-of-source builds if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) set(lines @@ -80,6 +80,8 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR AND NOT DEFINED PYBIND11_MASTER_ endif() set(pybind11_system "") + + set_property(GLOBAL PROPERTY USE_FOLDERS ON) else() set(PYBIND11_MASTER_PROJECT OFF) set(pybind11_system SYSTEM) @@ -89,6 +91,15 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) +set(PYBIND11_INTERNALS_VERSION + "" + CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") + +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() cmake_dependent_option( USE_PYTHON_INCLUDE_DIR @@ -105,6 +116,7 @@ set(PYBIND11_HEADERS include/pybind11/detail/descr.h include/pybind11/detail/init.h include/pybind11/detail/internals.h + include/pybind11/detail/type_caster_base.h include/pybind11/detail/typeid.h include/pybind11/attr.h include/pybind11/buffer_info.h @@ -114,8 +126,12 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h + include/pybind11/eigen/matrix.h + include/pybind11/eigen/tensor.h include/pybind11/embed.h include/pybind11/eval.h + include/pybind11/gil.h include/pybind11/iostream.h include/pybind11/functional.h include/pybind11/numpy.h @@ -123,7 +139,9 @@ set(PYBIND11_HEADERS include/pybind11/pybind11.h include/pybind11/pytypes.h include/pybind11/stl.h - include/pybind11/stl_bind.h) + include/pybind11/stl_bind.h + include/pybind11/stl/filesystem.h + include/pybind11/type_caster_pyobject_ptr.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) @@ -180,12 +198,19 @@ if(NOT TARGET pybind11_headers) target_compile_features(pybind11_headers INTERFACE cxx_inheriting_constructors cxx_user_literals cxx_right_angle_brackets) + if(NOT "${PYBIND11_INTERNALS_VERSION}" STREQUAL "") + target_compile_definitions( + pybind11_headers INTERFACE "PYBIND11_INTERNALS_VERSION=${PYBIND11_INTERNALS_VERSION}") + endif() else() # It is invalid to install a target twice, too. set(PYBIND11_INSTALL OFF) endif() include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") +# https://github.com/jtojnar/cmake-snips/#concatenating-paths-when-building-pkg-config-files +# TODO: cmake 3.20 adds the cmake_path() function, which obsoletes this snippet +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake") # Relative directory setting if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) @@ -200,6 +225,12 @@ if(PYBIND11_INSTALL) "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}" CACHE STRING "install path for pybind11Config.cmake") + if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(pybind11_INCLUDEDIR "${CMAKE_INSTALL_FULL_INCLUDEDIR}") + else() + set(pybind11_INCLUDEDIR "\$\{PACKAGE_PREFIX_DIR\}/${CMAKE_INSTALL_INCLUDEDIR}") + endif() + configure_package_config_file( tools/${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" INSTALL_DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) @@ -244,6 +275,16 @@ if(PYBIND11_INSTALL) NAMESPACE "pybind11::" DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + # pkg-config support + if(NOT prefix_for_pc_file) + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + endif() + join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") + # Uninstall target if(PYBIND11_MASTER_PROJECT) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in" diff --git a/MANIFEST.in b/MANIFEST.in index aed183e8..7ce83c55 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ +prune tests recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed -recursive-include pybind11 *.pyi include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg +include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg @@ -1,3 +1,7 @@ +# This project was upgraded with external_updater. +# Usage: tools/external_updater/updater.sh update python/pybind11 +# For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md + name: "pybind11" description: "pybind11 is a lightweight header-only library that exposes C++ types in Python and vice versa, mainly to create Python bindings of existing C++ code." third_party { @@ -9,11 +13,11 @@ third_party { type: GIT value: "https://github.com/pybind/pybind11.git" } - version: "v2.6.2" + version: "v2.11.0" license_type: NOTICE last_upgrade_date { - year: 2021 - month: 1 - day: 26 + year: 2023 + month: 8 + day: 22 } } @@ -3,7 +3,7 @@ **pybind11 — Seamless operability between C++11 and Python** -|Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |CI| |Build status| +|Latest Documentation Status| |Stable Documentation Status| |Gitter chat| |GitHub Discussions| |CI| |Build status| |Repology| |PyPI package| |Conda-forge| |Python Versions| @@ -13,19 +13,6 @@ .. start -.. warning:: - - Combining older versions of pybind11 (< 2.6.0) with Python 3.9.0 will - trigger undefined behavior that typically manifests as crashes during - interpreter shutdown (but could also destroy your data. **You have been - warned.**) - - We recommend that you update to the latest patch release of Python (3.9.1), - which includes a `fix <https://github.com/python/cpython/pull/22670>`_ - that resolves this problem. If you do use Python 3.9.0, please update to - the latest version of pybind11 (2.6.0 or newer), which includes a temporary - workaround specifically when Python 3.9.0 is detected at runtime. - **pybind11** is a lightweight header-only library that exposes C++ types in Python and vice versa, mainly to create Python bindings of existing @@ -45,9 +32,9 @@ this heavy machinery has become an excessively large and unnecessary dependency. Think of this library as a tiny self-contained version of Boost.Python -with everything stripped away that isn’t relevant for binding +with everything stripped away that isn't relevant for binding generation. Without comments, the core header files only require ~4K -lines of code and depend on Python (2.7 or 3.5+, or PyPy) and the C++ +lines of code and depend on Python (3.6+, or PyPy) and the C++ standard library. This compact implementation was possible thanks to some of the new C++11 language features (specifically: tuples, lambda functions and variadic templates). Since its creation, this library has @@ -91,8 +78,8 @@ Goodies In addition to the core functionality, pybind11 provides some extra goodies: -- Python 2.7, 3.5+, and PyPy/PyPy3 7.3 are supported with an - implementation-agnostic interface. +- Python 3.6+, and PyPy3 7.3 are supported with an implementation-agnostic + interface (pybind11 2.9 was the last version to support Python 2 and 3.5). - It is possible to bind C++11 lambda functions with captured variables. The lambda capture data is stored inside the resulting @@ -101,8 +88,8 @@ goodies: - pybind11 uses C++11 move constructors and move assignment operators whenever possible to efficiently transfer custom data types. -- It’s easy to expose the internal storage of custom data types through - Pythons’ buffer protocols. This is handy e.g. for fast conversion +- It's easy to expose the internal storage of custom data types through + Pythons' buffer protocols. This is handy e.g. for fast conversion between C++ matrix classes like Eigen and NumPy without expensive copy operations. @@ -110,7 +97,7 @@ goodies: transparently applied to all entries of one or more NumPy array arguments. -- Python’s slice-based access and assignment operations can be +- Python's slice-based access and assignment operations can be supported with just a few lines of code. - Everything is contained in just a few header files; there is no need @@ -119,7 +106,7 @@ goodies: - Binaries are generally smaller by a factor of at least 2 compared to equivalent bindings generated by Boost.Python. A recent pybind11 conversion of PyRosetta, an enormous Boost.Python binding project, - `reported <http://graylab.jhu.edu/RosettaCon2016/PyRosetta-4.pdf>`_ + `reported <https://graylab.jhu.edu/Sergey/2016.RosettaCon/PyRosetta-4.pdf>`_ a binary size reduction of **5.4x** and compile time reduction by **5.8x**. @@ -132,10 +119,10 @@ goodies: Supported compilers ------------------- -1. Clang/LLVM 3.3 or newer (for Apple Xcode’s clang, this is 5.0.0 or +1. Clang/LLVM 3.3 or newer (for Apple Xcode's clang, this is 5.0.0 or newer) 2. GCC 4.8 or newer -3. Microsoft Visual Studio 2015 Update 3 or newer +3. Microsoft Visual Studio 2017 or newer 4. Intel classic C++ compiler 18 or newer (ICC 20.2 tested in CI) 5. Cygwin/GCC (previously tested on 2.5.1) 6. NVCC (CUDA 11.0 tested in CI) @@ -147,9 +134,9 @@ About This project was created by `Wenzel Jakob <http://rgl.epfl.ch/people/wjakob>`_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, -Sylvain Corlay, Eric Cousineau, Ralf Grosse-Kunstleve, Trent Houliston, Axel +Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, -Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim +Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. We thank Google for a generous financial contribution to the continuous @@ -189,3 +176,5 @@ to the terms and conditions of this license. :target: https://repology.org/project/python:pybind11/versions .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/pybind11.svg :target: https://pypi.org/project/pybind11/ +.. |GitHub Discussions| image:: https://img.shields.io/static/v1?label=Discussions&message=Ask&color=blue&logo=github + :target: https://github.com/pybind/pybind11/discussions diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..3d74611f --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/docs/Doxyfile b/docs/Doxyfile index c8562952..09138db3 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -18,6 +18,4 @@ ALIASES += "endrst=\endverbatim" QUIET = YES WARNINGS = YES WARN_IF_UNDOCUMENTED = NO -PREDEFINED = DOXYGEN_SHOULD_SKIP_THIS \ - PY_MAJOR_VERSION=3 \ - PYBIND11_NOINLINE +PREDEFINED = PYBIND11_NOINLINE diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 00000000..7a49a6ac --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,3 @@ +.highlight .go { + color: #707070; +} diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css deleted file mode 100644 index 1071809f..00000000 --- a/docs/_static/theme_overrides.css +++ /dev/null @@ -1,11 +0,0 @@ -.wy-table-responsive table td, -.wy-table-responsive table th { - white-space: initial !important; -} -.rst-content table.docutils td { - vertical-align: top !important; -} -div[class^='highlight'] pre { - white-space: pre; - white-space: pre-wrap; -} diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index a779444c..8138cac6 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -26,7 +26,9 @@ The following Python snippet demonstrates the intended usage from the Python sid def __int__(self): return 123 + from example import print + print(A()) To register the necessary conversion routines, it is necessary to add an @@ -36,7 +38,7 @@ type is explicitly allowed. .. code-block:: cpp - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster<inty> { public: /** @@ -44,7 +46,7 @@ type is explicitly allowed. * function signatures and declares a local variable * 'value' of type inty */ - PYBIND11_TYPE_CASTER(inty, _("inty")); + PYBIND11_TYPE_CASTER(inty, const_name("inty")); /** * Conversion part 1 (Python->C++): convert a PyObject into a inty @@ -76,7 +78,7 @@ type is explicitly allowed. return PyLong_FromLong(src.long_value); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail .. note:: diff --git a/docs/advanced/cast/eigen.rst b/docs/advanced/cast/eigen.rst index e01472d5..a5c11a3f 100644 --- a/docs/advanced/cast/eigen.rst +++ b/docs/advanced/cast/eigen.rst @@ -52,7 +52,7 @@ can be mapped *and* if the numpy array is writeable (that is the passed variable will be transparently carried out directly on the ``numpy.ndarray``. -This means you can can write code such as the following and have it work as +This means you can write code such as the following and have it work as expected: .. code-block:: cpp @@ -112,7 +112,7 @@ example: .. code-block:: python a = MyClass() - m = a.get_matrix() # flags.writeable = True, flags.owndata = False + m = a.get_matrix() # flags.writeable = True, flags.owndata = False v = a.view_matrix() # flags.writeable = False, flags.owndata = False c = a.copy_matrix() # flags.writeable = True, flags.owndata = True # m[5,6] and v[5,6] refer to the same element, c[5,6] does not. @@ -203,7 +203,7 @@ adding the ``order='F'`` option when creating an array: .. code-block:: python - myarray = np.array(source, order='F') + myarray = np.array(source, order="F") Such an object will be passable to a bound function accepting an ``Eigen::Ref<MatrixXd>`` (or similar column-major Eigen type). diff --git a/docs/advanced/cast/overview.rst b/docs/advanced/cast/overview.rst index b0e32a52..011bd4c7 100644 --- a/docs/advanced/cast/overview.rst +++ b/docs/advanced/cast/overview.rst @@ -75,91 +75,96 @@ The following basic data types are supported out of the box (some may require an additional extension header to be included). To pass other data structures as arguments and return values, refer to the section on binding :ref:`classes`. -+------------------------------------+---------------------------+-------------------------------+ -| Data type | Description | Header file | -+====================================+===========================+===============================+ -| ``int8_t``, ``uint8_t`` | 8-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``int16_t``, ``uint16_t`` | 16-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``int32_t``, ``uint32_t`` | 32-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``int64_t``, ``uint64_t`` | 64-bit integers | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``ssize_t``, ``size_t`` | Platform-dependent size | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``float``, ``double`` | Floating point types | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``bool`` | Two-state Boolean type | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``char`` | Character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``char16_t`` | UTF-16 character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``char32_t`` | UTF-32 character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``wchar_t`` | Wide character literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const char *`` | UTF-8 string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const char16_t *`` | UTF-16 string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const char32_t *`` | UTF-32 string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``const wchar_t *`` | Wide string literal | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::string`` | STL dynamic UTF-8 string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::u16string`` | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::u32string`` | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::wstring`` | STL dynamic wide string | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::string_view``, | STL C++17 string views | :file:`pybind11/pybind11.h` | -| ``std::u16string_view``, etc. | | | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::pair<T1, T2>`` | Pair of two custom types | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::tuple<...>`` | Arbitrary tuple of types | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::reference_wrapper<...>`` | Reference type wrapper | :file:`pybind11/pybind11.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::complex<T>`` | Complex numbers | :file:`pybind11/complex.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::array<T, Size>`` | STL static array | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::vector<T>`` | STL dynamic array | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::deque<T>`` | STL double-ended queue | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::valarray<T>`` | STL value array | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::list<T>`` | STL linked list | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::map<T1, T2>`` | STL ordered map | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::unordered_map<T1, T2>`` | STL unordered map | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::set<T>`` | STL ordered set | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::unordered_set<T>`` | STL unordered set | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::optional<T>`` | STL optional type (C++17) | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::experimental::optional<T>`` | STL optional type (exp.) | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::variant<...>`` | Type-safe union (C++17) | :file:`pybind11/stl.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::function<...>`` | STL polymorphic function | :file:`pybind11/functional.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::chrono::duration<...>`` | STL time duration | :file:`pybind11/chrono.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``std::chrono::time_point<...>`` | STL date/time | :file:`pybind11/chrono.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``Eigen::Matrix<...>`` | Eigen: dense matrix | :file:`pybind11/eigen.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``Eigen::Map<...>`` | Eigen: mapped memory | :file:`pybind11/eigen.h` | -+------------------------------------+---------------------------+-------------------------------+ -| ``Eigen::SparseMatrix<...>`` | Eigen: sparse matrix | :file:`pybind11/eigen.h` | -+------------------------------------+---------------------------+-------------------------------+ ++------------------------------------+---------------------------+-----------------------------------+ +| Data type | Description | Header file | ++====================================+===========================+===================================+ +| ``int8_t``, ``uint8_t`` | 8-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``int16_t``, ``uint16_t`` | 16-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``int32_t``, ``uint32_t`` | 32-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``int64_t``, ``uint64_t`` | 64-bit integers | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``ssize_t``, ``size_t`` | Platform-dependent size | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``float``, ``double`` | Floating point types | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``bool`` | Two-state Boolean type | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``char`` | Character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``char16_t`` | UTF-16 character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``char32_t`` | UTF-32 character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``wchar_t`` | Wide character literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const char *`` | UTF-8 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const char16_t *`` | UTF-16 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const char32_t *`` | UTF-32 string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``const wchar_t *`` | Wide string literal | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::string`` | STL dynamic UTF-8 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::u16string`` | STL dynamic UTF-16 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::u32string`` | STL dynamic UTF-32 string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::wstring`` | STL dynamic wide string | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::string_view``, | STL C++17 string views | :file:`pybind11/pybind11.h` | +| ``std::u16string_view``, etc. | | | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::pair<T1, T2>`` | Pair of two custom types | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::tuple<...>`` | Arbitrary tuple of types | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::reference_wrapper<...>`` | Reference type wrapper | :file:`pybind11/pybind11.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::complex<T>`` | Complex numbers | :file:`pybind11/complex.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::array<T, Size>`` | STL static array | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::vector<T>`` | STL dynamic array | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::deque<T>`` | STL double-ended queue | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::valarray<T>`` | STL value array | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::list<T>`` | STL linked list | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::map<T1, T2>`` | STL ordered map | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::unordered_map<T1, T2>`` | STL unordered map | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::set<T>`` | STL ordered set | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::unordered_set<T>`` | STL unordered set | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::optional<T>`` | STL optional type (C++17) | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::experimental::optional<T>`` | STL optional type (exp.) | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::variant<...>`` | Type-safe union (C++17) | :file:`pybind11/stl.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::filesystem::path<T>`` | STL path (C++17) [#]_ | :file:`pybind11/stl/filesystem.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::function<...>`` | STL polymorphic function | :file:`pybind11/functional.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::chrono::duration<...>`` | STL time duration | :file:`pybind11/chrono.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``std::chrono::time_point<...>`` | STL date/time | :file:`pybind11/chrono.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``Eigen::Matrix<...>`` | Eigen: dense matrix | :file:`pybind11/eigen.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``Eigen::Map<...>`` | Eigen: mapped memory | :file:`pybind11/eigen.h` | ++------------------------------------+---------------------------+-----------------------------------+ +| ``Eigen::SparseMatrix<...>`` | Eigen: sparse matrix | :file:`pybind11/eigen.h` | ++------------------------------------+---------------------------+-----------------------------------+ + +.. [#] ``std::filesystem::path`` is converted to ``pathlib.Path`` and + ``os.PathLike`` is converted to ``std::filesystem::path``. diff --git a/docs/advanced/cast/stl.rst b/docs/advanced/cast/stl.rst index b8622ee0..03d49b29 100644 --- a/docs/advanced/cast/stl.rst +++ b/docs/advanced/cast/stl.rst @@ -42,7 +42,7 @@ types: .. code-block:: cpp // `boost::optional` as an example -- can be any `std::optional`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <typename T> struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {}; }} @@ -54,7 +54,7 @@ for custom variant types: .. code-block:: cpp // `boost::variant` as an example -- can be any `std::variant`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <typename... Ts> struct type_caster<boost::variant<Ts...>> : variant_caster<boost::variant<Ts...>> {}; @@ -66,7 +66,7 @@ for custom variant types: return boost::apply_visitor(args...); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail The ``visit_helper`` specialization is not required if your ``name::variant`` provides a ``name::visit()`` function. For any other function name, the specialization must be @@ -87,8 +87,6 @@ included to tell pybind11 how to visit the variant. pybind11 only supports the modern implementation of ``boost::variant`` which makes use of variadic templates. This requires Boost 1.56 or newer. - Additionally, on Windows, MSVC 2017 is required because ``boost::variant`` - falls back to the old non-variadic implementation on MSVC 2015. .. _opaque: diff --git a/docs/advanced/cast/strings.rst b/docs/advanced/cast/strings.rst index e25701ec..271716b4 100644 --- a/docs/advanced/cast/strings.rst +++ b/docs/advanced/cast/strings.rst @@ -1,14 +1,6 @@ Strings, bytes and Unicode conversions ###################################### -.. note:: - - This section discusses string handling in terms of Python 3 strings. For - Python 2.7, replace all occurrences of ``str`` with ``unicode`` and - ``bytes`` with ``str``. Python 2.7 users may find it best to use ``from - __future__ import unicode_literals`` to avoid unintentionally using ``str`` - instead of ``unicode``. - Passing Python strings to C++ ============================= @@ -36,13 +28,13 @@ everywhere <http://utf8everywhere.org/>`_. } ); -.. code-block:: python +.. code-block:: pycon - >>> utf8_test('🎂') + >>> utf8_test("🎂") utf-8 is icing on the cake. 🎂 - >>> utf8_charptr('🍕') + >>> utf8_charptr("🍕") My favorite food is 🍕 @@ -58,9 +50,9 @@ Passing bytes to C++ -------------------- A Python ``bytes`` object will be passed to C++ functions that accept -``std::string`` or ``char*`` *without* conversion. On Python 3, in order to -make a function *only* accept ``bytes`` (and not ``str``), declare it as taking -a ``py::bytes`` argument. +``std::string`` or ``char*`` *without* conversion. In order to make a function +*only* accept ``bytes`` (and not ``str``), declare it as taking a ``py::bytes`` +argument. Returning C++ strings to Python @@ -80,7 +72,7 @@ raise a ``UnicodeDecodeError``. } ); -.. code-block:: python +.. code-block:: pycon >>> isinstance(example.std_string_return(), str) True @@ -109,19 +101,23 @@ conversion has the same overhead as implicit conversion. m.def("str_output", []() { std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1 - py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length()); - return py_s; + py::handle py_s = PyUnicode_DecodeLatin1(s.data(), s.length(), nullptr); + if (!py_s) { + throw py::error_already_set(); + } + return py::reinterpret_steal<py::str>(py_s); } ); -.. code-block:: python +.. code-block:: pycon >>> str_output() 'Send your résumé to Alice in HR' The `Python C API <https://docs.python.org/3/c-api/unicode.html#built-in-codecs>`_ provides -several built-in codecs. +several built-in codecs. Note that these all return *new* references, so +use :cpp:func:`reinterpret_steal` when converting them to a :cpp:class:`str`. One could also use a third party encoding library such as libiconv to transcode @@ -143,7 +139,7 @@ returned to Python as ``bytes``, then one can return the data as a } ); -.. code-block:: python +.. code-block:: pycon >>> example.return_bytes() b'\xba\xd0\xba\xd0' @@ -160,7 +156,7 @@ encoding, but cannot convert ``std::string`` back to ``bytes`` implicitly. } ); -.. code-block:: python +.. code-block:: pycon >>> isinstance(example.asymmetry(b"have some bytes"), str) True @@ -204,11 +200,6 @@ decoded to Python ``str``. } ); -.. warning:: - - Wide character strings may not work as described on Python 2.7 or Python - 3.3 compiled with ``--enable-unicode=ucs2``. - Strings in multibyte encodings such as Shift-JIS must transcoded to a UTF-8/16/32 before being returned to Python. @@ -229,16 +220,16 @@ character. m.def("pass_char", [](char c) { return c; }); m.def("pass_wchar", [](wchar_t w) { return w; }); -.. code-block:: python +.. code-block:: pycon - >>> example.pass_char('A') + >>> example.pass_char("A") 'A' While C++ will cast integers to character types (``char c = 0x65;``), pybind11 does not convert Python integers to characters implicitly. The Python function ``chr()`` can be used to convert integers to characters. -.. code-block:: python +.. code-block:: pycon >>> example.pass_char(0x65) TypeError @@ -259,17 +250,17 @@ a combining acute accent). The combining character will be lost if the two-character sequence is passed as an argument, even though it renders as a single grapheme. -.. code-block:: python +.. code-block:: pycon - >>> example.pass_wchar('é') + >>> example.pass_wchar("é") 'é' - >>> combining_e_acute = 'e' + '\u0301' + >>> combining_e_acute = "e" + "\u0301" >>> combining_e_acute 'é' - >>> combining_e_acute == 'é' + >>> combining_e_acute == "é" False >>> example.pass_wchar(combining_e_acute) @@ -278,9 +269,9 @@ single grapheme. Normalizing combining characters before passing the character literal to C++ may resolve *some* of these issues: -.. code-block:: python +.. code-block:: pycon - >>> example.pass_wchar(unicodedata.normalize('NFC', combining_e_acute)) + >>> example.pass_wchar(unicodedata.normalize("NFC", combining_e_acute)) 'é' In some languages (Thai for example), there are `graphemes that cannot be diff --git a/docs/advanced/classes.rst b/docs/advanced/classes.rst index 49279020..01a490b7 100644 --- a/docs/advanced/classes.rst +++ b/docs/advanced/classes.rst @@ -9,7 +9,7 @@ that you are already familiar with the basics from :doc:`/classes`. Overriding virtual functions in Python ====================================== -Suppose that a C++ class or interface has a virtual function that we'd like to +Suppose that a C++ class or interface has a virtual function that we'd like to override from within Python (we'll focus on the class ``Animal``; ``Dog`` is given as a specific example of how one would do this with traditional C++ code). @@ -133,14 +133,14 @@ a virtual method call. >>> from example import * >>> d = Dog() >>> call_go(d) - u'woof! woof! woof! ' + 'woof! woof! woof! ' >>> class Cat(Animal): ... def go(self, n_times): - ... return "meow! " * n_times + ... return "meow! " * n_times ... >>> c = Cat() >>> call_go(c) - u'meow! meow! meow! ' + 'meow! meow! meow! ' If you are defining a custom constructor in a derived Python class, you *must* ensure that you explicitly call the bound C++ constructor using ``__init__``, @@ -159,8 +159,9 @@ Here is an example: class Dachshund(Dog): def __init__(self, name): - Dog.__init__(self) # Without this, a TypeError is raised. + Dog.__init__(self) # Without this, a TypeError is raised. self.name = name + def bark(self): return "yap!" @@ -259,7 +260,7 @@ override the ``name()`` method): .. note:: - Note the trailing commas in the ``PYBIND11_OVERIDE`` calls to ``name()`` + Note the trailing commas in the ``PYBIND11_OVERRIDE`` calls to ``name()`` and ``bark()``. These are needed to portably implement a trampoline for a function that does not take any arguments. For functions that take a nonzero number of arguments, the trailing comma must be omitted. @@ -804,7 +805,7 @@ to bind these two functions: } )); -The ``__setstate__`` part of the ``py::picke()`` definition follows the same +The ``__setstate__`` part of the ``py::pickle()`` definition follows the same rules as the single-argument version of ``py::init()``. The return type can be a value, pointer or holder type. See :ref:`custom_constructors` for details. @@ -812,26 +813,21 @@ An instance can now be pickled as follows: .. code-block:: python - try: - import cPickle as pickle # Use cPickle on Python 2.7 - except ImportError: - import pickle + import pickle p = Pickleable("test_value") p.setExtra(15) - data = pickle.dumps(p, 2) + data = pickle.dumps(p) .. note:: - Note that only the cPickle module is supported on Python 2.7. - - The second argument to ``dumps`` is also crucial: it selects the pickle - protocol version 2, since the older version 1 is not supported. Newer - versions are also fine—for instance, specify ``-1`` to always use the - latest available version. Beware: failure to follow these instructions - will cause important pybind11 memory allocation routines to be skipped - during unpickling, which will likely lead to memory corruption and/or - segmentation faults. + If given, the second argument to ``dumps`` must be 2 or larger - 0 and 1 are + not supported. Newer versions are also fine; for instance, specify ``-1`` to + always use the latest available version. Beware: failure to follow these + instructions will cause important pybind11 memory allocation routines to be + skipped during unpickling, which will likely lead to memory corruption + and/or segmentation faults. Python defaults to version 3 (Python 3-3.7) and + version 4 for Python 3.8+. .. seealso:: @@ -848,11 +844,9 @@ Python normally uses references in assignments. Sometimes a real copy is needed to prevent changing all copies. The ``copy`` module [#f5]_ provides these capabilities. -On Python 3, a class with pickle support is automatically also (deep)copy +A class with pickle support is automatically also (deep)copy compatible. However, performance can be improved by adding custom -``__copy__`` and ``__deepcopy__`` methods. With Python 2.7, these custom methods -are mandatory for (deep)copy compatibility, because pybind11 only supports -cPickle. +``__copy__`` and ``__deepcopy__`` methods. For simple classes (deep)copy can be enabled by using the copy constructor, which should look as follows: @@ -1124,13 +1118,6 @@ described trampoline: py::class_<A, Trampoline>(m, "A") // <-- `Trampoline` here .def("foo", &Publicist::foo); // <-- `Publicist` here, not `Trampoline`! -.. note:: - - MSVC 2015 has a compiler bug (fixed in version 2017) which - requires a more explicit function binding in the form of - ``.def("foo", static_cast<int (A::*)() const>(&Publicist::foo));`` - where ``int (A::*)() const`` is the type of ``A::foo``. - Binding final classes ===================== @@ -1153,12 +1140,65 @@ error: >>> class PyFinalChild(IsFinal): ... pass + ... TypeError: type 'IsFinal' is not an acceptable base type .. note:: This attribute is currently ignored on PyPy .. versionadded:: 2.6 +Binding classes with template parameters +======================================== + +pybind11 can also wrap classes that have template parameters. Consider these classes: + +.. code-block:: cpp + + struct Cat {}; + struct Dog {}; + + template <typename PetType> + struct Cage { + Cage(PetType& pet); + PetType& get(); + }; + +C++ templates may only be instantiated at compile time, so pybind11 can only +wrap instantiated templated classes. You cannot wrap a non-instantiated template: + +.. code-block:: cpp + + // BROKEN (this will not compile) + py::class_<Cage>(m, "Cage"); + .def("get", &Cage::get); + +You must explicitly specify each template/type combination that you want to +wrap separately. + +.. code-block:: cpp + + // ok + py::class_<Cage<Cat>>(m, "CatCage") + .def("get", &Cage<Cat>::get); + + // ok + py::class_<Cage<Dog>>(m, "DogCage") + .def("get", &Cage<Dog>::get); + +If your class methods have template parameters you can wrap those as well, +but once again each instantiation must be explicitly specified: + +.. code-block:: cpp + + typename <typename T> + struct MyClass { + template <typename V> + T fn(V v); + }; + + py::class<MyClass<int>>(m, "MyClassT") + .def("fn", &MyClass<int>::fn<std::string>); + Custom automatic downcasters ============================ @@ -1188,7 +1228,7 @@ whether a downcast is safe, you can proceed by specializing the std::string bark() const { return sound; } }; - namespace pybind11 { + namespace PYBIND11_NAMESPACE { template<> struct polymorphic_type_hook<Pet> { static const void *get(const Pet *src, const std::type_info*& type) { // note that src may be nullptr @@ -1199,7 +1239,7 @@ whether a downcast is safe, you can proceed by specializing the return src; } }; - } // namespace pybind11 + } // namespace PYBIND11_NAMESPACE When pybind11 wants to convert a C++ pointer of type ``Base*`` to a Python object, it calls ``polymorphic_type_hook<Base>::get()`` to @@ -1247,7 +1287,7 @@ Accessing the type object You can get the type object from a C++ class that has already been registered using: -.. code-block:: python +.. code-block:: cpp py::type T_py = py::type::of<T>(); @@ -1259,3 +1299,37 @@ object, just like ``type(ob)`` in Python. Other types, like ``py::type::of<int>()``, do not work, see :ref:`type-conversions`. .. versionadded:: 2.6 + +Custom type setup +================= + +For advanced use cases, such as enabling garbage collection support, you may +wish to directly manipulate the ``PyHeapTypeObject`` corresponding to a +``py::class_`` definition. + +You can do that using ``py::custom_type_setup``: + +.. code-block:: cpp + + struct OwnsPythonObjects { + py::object value = py::none(); + }; + py::class_<OwnsPythonObjects> cls( + m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + auto *type = &heap_type->ht_type; + type->tp_flags |= Py_TPFLAGS_HAVE_GC; + type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { + auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base)); + Py_VISIT(self.value.ptr()); + return 0; + }; + type->tp_clear = [](PyObject *self_base) { + auto &self = py::cast<OwnsPythonObjects&>(py::handle(self_base)); + self.value = py::none(); + return 0; + }; + })); + cls.def(py::init<>()); + cls.def_readwrite("value", &OwnsPythonObjects::value); + +.. versionadded:: 2.8 diff --git a/docs/advanced/embedding.rst b/docs/advanced/embedding.rst index dfdaad2d..e6a1686f 100644 --- a/docs/advanced/embedding.rst +++ b/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` @@ -40,15 +40,15 @@ The essential structure of the ``main.cpp`` file looks like this: } The interpreter must be initialized before using any Python API, which includes -all the functions and classes in pybind11. The RAII guard class `scoped_interpreter` +all the functions and classes in pybind11. The RAII guard class ``scoped_interpreter`` takes care of the interpreter lifetime. After the guard is destroyed, the interpreter shuts down and clears its memory. No Python functions can be called after this. Executing Python code ===================== -There are a few different ways to run Python code. One option is to use `eval`, -`exec` or `eval_file`, as explained in :ref:`eval`. Here is a quick example in +There are a few different ways to run Python code. One option is to use ``eval``, +``exec`` or ``eval_file``, as explained in :ref:`eval`. Here is a quick example in the context of an executable with an embedded interpreter: .. code-block:: cpp @@ -108,7 +108,7 @@ The two approaches can also be combined: Importing modules ================= -Python modules can be imported using `module_::import()`: +Python modules can be imported using ``module_::import()``: .. code-block:: cpp @@ -122,6 +122,7 @@ embedding the interpreter. This makes it easy to import local Python files: """calc.py located in the working directory""" + def add(i, j): return i + j @@ -133,7 +134,7 @@ embedding the interpreter. This makes it easy to import local Python files: int n = result.cast<int>(); assert(n == 3); -Modules can be reloaded using `module_::reload()` if the source is modified e.g. +Modules can be reloaded using ``module_::reload()`` if the source is modified e.g. by an external process. This can be useful in scenarios where the application imports a user defined data processing script which needs to be updated after changes by the user. Note that this function does not reload modules recursively. @@ -143,7 +144,7 @@ changes by the user. Note that this function does not reload modules recursively Adding embedded modules ======================= -Embedded binary modules can be added using the `PYBIND11_EMBEDDED_MODULE` macro. +Embedded binary modules can be added using the ``PYBIND11_EMBEDDED_MODULE`` macro. Note that the definition must be placed at global scope. They can be imported like any other module. @@ -169,7 +170,7 @@ like any other module. Unlike extension modules where only a single binary module can be created, on the embedded side an unlimited number of modules can be added using multiple -`PYBIND11_EMBEDDED_MODULE` definitions (as long as they have unique names). +``PYBIND11_EMBEDDED_MODULE`` definitions (as long as they have unique names). These modules are added to Python's list of builtins, so they can also be imported in pure Python files loaded by the interpreter. Everything interacts @@ -215,9 +216,9 @@ naturally: Interpreter lifetime ==================== -The Python interpreter shuts down when `scoped_interpreter` is destroyed. After +The Python interpreter shuts down when ``scoped_interpreter`` is destroyed. After this, creating a new instance will restart the interpreter. Alternatively, the -`initialize_interpreter` / `finalize_interpreter` pair of functions can be used +``initialize_interpreter`` / ``finalize_interpreter`` pair of functions can be used to directly set the state at any time. Modules created with pybind11 can be safely re-initialized after the interpreter @@ -229,8 +230,8 @@ global data. All the details can be found in the CPython documentation. .. warning:: - Creating two concurrent `scoped_interpreter` guards is a fatal error. So is - calling `initialize_interpreter` for a second time after the interpreter + Creating two concurrent ``scoped_interpreter`` guards is a fatal error. So is + calling ``initialize_interpreter`` for a second time after the interpreter has already been initialized. Do not use the raw CPython API functions ``Py_Initialize`` and @@ -241,7 +242,7 @@ global data. All the details can be found in the CPython documentation. Sub-interpreter support ======================= -Creating multiple copies of `scoped_interpreter` is not possible because it +Creating multiple copies of ``scoped_interpreter`` is not possible because it represents the main Python interpreter. Sub-interpreters are something different and they do permit the existence of multiple interpreters. This is an advanced feature of the CPython API and should be handled with care. pybind11 does not @@ -257,5 +258,5 @@ We'll just mention a couple of caveats the sub-interpreters support in pybind11: 2. Managing multiple threads, multiple interpreters and the GIL can be challenging and there are several caveats here, even within the pure CPython API (please refer to the Python docs for details). As for - pybind11, keep in mind that `gil_scoped_release` and `gil_scoped_acquire` + pybind11, keep in mind that ``gil_scoped_release`` and ``gil_scoped_acquire`` do not take sub-interpreters into account. diff --git a/docs/advanced/exceptions.rst b/docs/advanced/exceptions.rst index 7a4d6cbc..53981dc0 100644 --- a/docs/advanced/exceptions.rst +++ b/docs/advanced/exceptions.rst @@ -56,13 +56,15 @@ at its exception handler. +--------------------------------------+--------------------------------------+ | :class:`pybind11::buffer_error` | ``BufferError`` | +--------------------------------------+--------------------------------------+ -| :class:`pybind11::import_error` | ``import_error`` | +| :class:`pybind11::import_error` | ``ImportError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::attribute_error` | ``AttributeError`` | +--------------------------------------+--------------------------------------+ | Any other exception | ``RuntimeError`` | +--------------------------------------+--------------------------------------+ Exception translation is not bidirectional. That is, *catching* the C++ -exceptions defined above above will not trap exceptions that originate from +exceptions defined above will not trap exceptions that originate from Python. For that, catch :class:`pybind11::error_already_set`. See :ref:`below <handling_python_exceptions_cpp>` for further details. @@ -75,9 +77,10 @@ Registering custom translators If the default exception conversion policy described above is insufficient, pybind11 also provides support for registering custom exception translators. -To register a simple exception conversion that translates a C++ exception into -a new Python exception using the C++ exception's ``what()`` method, a helper -function is available: +Similar to pybind11 classes, exception translators can be local to the module +they are defined in or global to the entire python session. To register a simple +exception conversion that translates a C++ exception into a new Python exception +using the C++ exception's ``what()`` method, a helper function is available: .. code-block:: cpp @@ -87,29 +90,39 @@ This call creates a Python exception class with the name ``PyExp`` in the given module and automatically converts any encountered exceptions of type ``CppExp`` into Python exceptions of type ``PyExp``. +A matching function is available for registering a local exception translator: + +.. code-block:: cpp + + py::register_local_exception<CppExp>(module, "PyExp"); + + It is possible to specify base class for the exception using the third -parameter, a `handle`: +parameter, a ``handle``: .. code-block:: cpp py::register_exception<CppExp>(module, "PyExp", PyExc_RuntimeError); + py::register_local_exception<CppExp>(module, "PyExp", PyExc_RuntimeError); -Then `PyExp` can be caught both as `PyExp` and `RuntimeError`. +Then ``PyExp`` can be caught both as ``PyExp`` and ``RuntimeError``. The class objects of the built-in Python exceptions are listed in the Python documentation on `Standard Exceptions <https://docs.python.org/3/c-api/exceptions.html#standard-exceptions>`_. -The default base class is `PyExc_Exception`. +The default base class is ``PyExc_Exception``. -When more advanced exception translation is needed, the function -``py::register_exception_translator(translator)`` can be used to register +When more advanced exception translation is needed, the functions +``py::register_exception_translator(translator)`` and +``py::register_local_exception_translator(translator)`` can be used to register functions that can translate arbitrary exception types (and which may include -additional logic to do so). The function takes a stateless callable (e.g. a +additional logic to do so). The functions takes a stateless callable (e.g. a function pointer or a lambda function without captured variables) with the call signature ``void(std::exception_ptr)``. When a C++ exception is thrown, the registered exception translators are tried in reverse order of registration (i.e. the last registered translator gets the -first shot at handling the exception). +first shot at handling the exception). All local translators will be tried +before a global translator is tried. Inside the translator, ``std::rethrow_exception`` should be used within a try block to re-throw the exception. One or more catch clauses to catch @@ -164,6 +177,60 @@ section. may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. + Note that ``libc++`` and ``libstdc++`` `behave differently under macOS + <https://stackoverflow.com/questions/19496643/using-clang-fvisibility-hidden-and-typeinfo-and-type-erasure/28827430>`_ + with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI + boundaries need to be explicitly exported, as exercised in + ``tests/test_exceptions.h``. See also: + "Problems with C++ exceptions" under `GCC Wiki <https://gcc.gnu.org/wiki/Visibility>`_. + + +Local vs Global Exception Translators +===================================== + +When a global exception translator is registered, it will be applied across all +modules in the reverse order of registration. This can create behavior where the +order of module import influences how exceptions are translated. + +If module1 has the following translator: + +.. code-block:: cpp + + py::register_exception_translator([](std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (const std::invalid_argument &e) { + PyErr_SetString("module1 handled this") + } + } + +and module2 has the following similar translator: + +.. code-block:: cpp + + py::register_exception_translator([](std::exception_ptr p) { + try { + if (p) std::rethrow_exception(p); + } catch (const std::invalid_argument &e) { + PyErr_SetString("module2 handled this") + } + } + +then which translator handles the invalid_argument will be determined by the +order that module1 and module2 are imported. Since exception translators are +applied in the reverse order of registration, which ever module was imported +last will "win" and that translator will be applied. + +If there are multiple pybind11 modules that share exception types (either +standard built-in or custom) loaded into a single python instance and +consistent error handling behavior is needed, then local translators should be +used. + +Changing the previous example to use ``register_local_exception_translator`` +would mean that when invalid_argument is thrown in the module2 code, the +module2 translator will always handle it, while in module1, the module1 +translator will do the same. + .. _handling_python_exceptions_cpp: Handling exceptions from Python in C++ @@ -261,6 +328,34 @@ Alternately, to ignore the error, call `PyErr_Clear Any Python error must be thrown or cleared, or Python/pybind11 will be left in an invalid state. +Chaining exceptions ('raise from') +================================== + +Python has a mechanism for indicating that exceptions were caused by other +exceptions: + +.. code-block:: py + + try: + print(1 / 0) + except Exception as exc: + raise RuntimeError("could not divide by zero") from exc + +To do a similar thing in pybind11, you can use the ``py::raise_from`` function. It +sets the current python error indicator, so to continue propagating the exception +you should ``throw py::error_already_set()``. + +.. code-block:: cpp + + try { + py::eval("print(1 / 0")); + } catch (py::error_already_set &e) { + py::raise_from(e, PyExc_RuntimeError, "could not divide by zero"); + throw py::error_already_set(); + } + +.. versionadded:: 2.8 + .. _unraisable_exceptions: Handling unraisable exceptions diff --git a/docs/advanced/functions.rst b/docs/advanced/functions.rst index 55a40a5b..69e3d8a1 100644 --- a/docs/advanced/functions.rst +++ b/docs/advanced/functions.rst @@ -50,7 +50,7 @@ implied transfer of ownership, i.e.: .. code-block:: cpp - m.def("get_data", &get_data, return_value_policy::reference); + m.def("get_data", &get_data, py::return_value_policy::reference); On the other hand, this is not the right policy for many other situations, where ignoring ownership could lead to resource leaks. @@ -90,17 +90,18 @@ The following table provides an overview of available policies: | | return value is referenced by Python. This is the default policy for | | | property getters created via ``def_property``, ``def_readwrite``, etc. | +--------------------------------------------------+----------------------------------------------------------------------------+ -| :enum:`return_value_policy::automatic` | **Default policy.** This policy falls back to the policy | +| :enum:`return_value_policy::automatic` | This policy falls back to the policy | | | :enum:`return_value_policy::take_ownership` when the return value is a | | | pointer. Otherwise, it uses :enum:`return_value_policy::move` or | | | :enum:`return_value_policy::copy` for rvalue and lvalue references, | | | respectively. See above for a description of what all of these different | -| | policies do. | +| | policies do. This is the default policy for ``py::class_``-wrapped types. | +--------------------------------------------------+----------------------------------------------------------------------------+ | :enum:`return_value_policy::automatic_reference` | As above, but use policy :enum:`return_value_policy::reference` when the | | | return value is a pointer. This is the default conversion policy for | | | function arguments when calling Python functions manually from C++ code | -| | (i.e. via handle::operator()). You probably won't need to use this. | +| | (i.e. via ``handle::operator()``) and the casters in ``pybind11/stl.h``. | +| | You probably won't need to use this explicitly. | +--------------------------------------------------+----------------------------------------------------------------------------+ Return value policies can also be applied to properties: @@ -119,7 +120,7 @@ targeted arguments can be passed through the :class:`cpp_function` constructor: .. code-block:: cpp class_<MyClass>(m, "MyClass") - .def_property("data" + .def_property("data", py::cpp_function(&MyClass::getData, py::return_value_policy::copy), py::cpp_function(&MyClass::setData) ); @@ -182,6 +183,9 @@ relies on the ability to create a *weak reference* to the nurse object. When the nurse object is not a pybind11-registered type and does not support weak references, an exception will be thrown. +If you use an incorrect argument index, you will get a ``RuntimeError`` saying +``Could not activate keep_alive!``. You should review the indices you're using. + Consider the following example: here, the binding code for a list append operation ties the lifetime of the newly added element to the underlying container: @@ -228,7 +232,7 @@ is equivalent to the following pseudocode: }); The only requirement is that ``T`` is default-constructible, but otherwise any -scope guard will work. This is very useful in combination with `gil_scoped_release`. +scope guard will work. This is very useful in combination with ``gil_scoped_release``. See :ref:`gil`. Multiple guards can also be specified as ``py::call_guard<T1, T2, T3...>``. The @@ -251,7 +255,7 @@ For instance, the following statement iterates over a Python ``dict``: .. code-block:: cpp - void print_dict(py::dict dict) { + void print_dict(const py::dict& dict) { /* Easily interact with Python types */ for (auto item : dict) std::cout << "key=" << std::string(py::str(item.first)) << ", " @@ -268,7 +272,7 @@ And used in Python as usual: .. code-block:: pycon - >>> print_dict({'foo': 123, 'bar': 'hello'}) + >>> print_dict({"foo": 123, "bar": "hello"}) key=foo, value=123 key=bar, value=hello @@ -289,7 +293,7 @@ Such functions can also be created using pybind11: .. code-block:: cpp - void generic(py::args args, py::kwargs kwargs) { + void generic(py::args args, const py::kwargs& kwargs) { /// .. do something with args if (kwargs) /// .. do something with kwargs @@ -302,8 +306,9 @@ The class ``py::args`` derives from ``py::tuple`` and ``py::kwargs`` derives from ``py::dict``. You may also use just one or the other, and may combine these with other -arguments as long as the ``py::args`` and ``py::kwargs`` arguments are the last -arguments accepted by the function. +arguments. Note, however, that ``py::kwargs`` must always be the last argument +of the function, and ``py::args`` implies that any further arguments are +keyword-only (see :ref:`keyword_only_arguments`). Please refer to the other examples for details on how to iterate over these, and on how to cast their entries into C++ objects. A demonstration is also @@ -362,10 +367,12 @@ like so: py::class_<MyClass>("MyClass") .def("myFunction", py::arg("arg") = static_cast<SomeType *>(nullptr)); +.. _keyword_only_arguments: + Keyword-only arguments ====================== -Python 3 introduced keyword-only arguments by specifying an unnamed ``*`` +Python implements keyword-only arguments by specifying an unnamed ``*`` argument in a function definition: .. code-block:: python @@ -373,10 +380,11 @@ argument in a function definition: def f(a, *, b): # a can be positional or via keyword; b must be via keyword pass + f(a=1, b=2) # good f(b=2, a=1) # good - f(1, b=2) # good - f(1, 2) # TypeError: f() takes 1 positional argument but 2 were given + f(1, b=2) # good + f(1, 2) # TypeError: f() takes 1 positional argument but 2 were given Pybind11 provides a ``py::kw_only`` object that allows you to implement the same behaviour by specifying the object between positional and keyword-only @@ -387,11 +395,19 @@ argument annotations when registering the function: m.def("f", [](int a, int b) { /* ... */ }, py::arg("a"), py::kw_only(), py::arg("b")); -Note that you currently cannot combine this with a ``py::args`` argument. This -feature does *not* require Python 3 to work. - .. versionadded:: 2.6 +A ``py::args`` argument implies that any following arguments are keyword-only, +as if ``py::kw_only()`` had been specified in the same relative location of the +argument list as the ``py::args`` argument. The ``py::kw_only()`` may be +included to be explicit about this, but is not required. + +.. versionchanged:: 2.9 + This can now be combined with ``py::args``. Before, ``py::args`` could only + occur at the end of the argument list, or immediately before a ``py::kwargs`` + argument at the end. + + Positional-only arguments ========================= @@ -561,3 +577,38 @@ prefers earlier-defined overloads to later-defined ones. .. versionadded:: 2.6 The ``py::prepend()`` tag. + +Binding functions with template parameters +========================================== + +You can bind functions that have template parameters. Here's a function: + +.. code-block:: cpp + + template <typename T> + void set(T t); + +C++ templates cannot be instantiated at runtime, so you cannot bind the +non-instantiated function: + +.. code-block:: cpp + + // BROKEN (this will not compile) + m.def("set", &set); + +You must bind each instantiated function template separately. You may bind +each instantiation with the same name, which will be treated the same as +an overloaded function: + +.. code-block:: cpp + + m.def("set", &set<int>); + m.def("set", &set<std::string>); + +Sometimes it's more clear to bind them with separate names, which is also +an option: + +.. code-block:: cpp + + m.def("setInt", &set<int>); + m.def("setString", &set<std::string>); diff --git a/docs/advanced/misc.rst b/docs/advanced/misc.rst index b3f3b226..805ec838 100644 --- a/docs/advanced/misc.rst +++ b/docs/advanced/misc.rst @@ -39,15 +39,42 @@ The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds. Global Interpreter Lock (GIL) ============================= -When calling a C++ function from Python, the GIL is always held. +The Python C API dictates that the Global Interpreter Lock (GIL) must always +be held by the current thread to safely access Python objects. As a result, +when Python calls into C++ via pybind11 the GIL must be held, and pybind11 +will never implicitly release the GIL. + +.. code-block:: cpp + + void my_function() { + /* GIL is held when this function is called from Python */ + } + + PYBIND11_MODULE(example, m) { + m.def("my_function", &my_function); + } + +pybind11 will ensure that the GIL is held when it knows that it is calling +Python code. For example, if a Python callback is passed to C++ code via +``std::function``, when C++ code calls the function the built-in wrapper +will acquire the GIL before calling the Python callback. Similarly, the +``PYBIND11_OVERRIDE`` family of macros will acquire the GIL before calling +back into Python. + +When writing C++ code that is called from other C++ code, if that code accesses +Python state, it must explicitly acquire and release the GIL. + The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ function call. In this way, long-running C++ code can be parallelized using -multiple Python threads. Taking :ref:`overriding_virtuals` as an example, this +multiple Python threads, **but great care must be taken** when any +:class:`gil_scoped_release` appear: if there is any way that the C++ code +can access Python objects, :class:`gil_scoped_acquire` should be used to +reacquire the GIL. Taking :ref:`overriding_virtuals` as an example, this could be realized as follows (important changes highlighted): .. code-block:: cpp - :emphasize-lines: 8,9,31,32 + :emphasize-lines: 8,30,31 class PyAnimal : public Animal { public: @@ -56,9 +83,7 @@ could be realized as follows (important changes highlighted): /* Trampoline (need one for each virtual function) */ std::string go(int n_times) { - /* Acquire GIL before calling Python code */ - py::gil_scoped_acquire acquire; - + /* PYBIND11_OVERRIDE_PURE will acquire the GIL before accessing Python state */ PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ @@ -78,13 +103,14 @@ could be realized as follows (important changes highlighted): .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { - /* Release GIL before calling into (potentially long-running) C++ code */ + // GIL is held when called from Python code. Release GIL before + // calling into (potentially long-running) C++ code py::gil_scoped_release release; return call_go(animal); }); } -The ``call_go`` wrapper can also be simplified using the `call_guard` policy +The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy (see :ref:`call_policies`) which yields the same result: .. code-block:: cpp @@ -92,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the `call_guard` policy m.def("call_go", &call_go, py::call_guard<py::gil_scoped_release>()); +Common Sources Of Global Interpreter Lock Errors +================================================================== + +Failing to properly hold the Global Interpreter Lock (GIL) is one of the +more common sources of bugs within code that uses pybind11. If you are +running into GIL related errors, we highly recommend you consult the +following checklist. + +- Do you have any global variables that are pybind11 objects or invoke + pybind11 functions in either their constructor or destructor? You are generally + not allowed to invoke any Python function in a global static context. We recommend + using lazy initialization and then intentionally leaking at the end of the program. + +- Do you have any pybind11 objects that are members of other C++ structures? One + commonly overlooked requirement is that pybind11 objects have to increase their reference count + whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke + the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very + tricky to track for complicated programs Think carefully when you make a pybind11 object + a member in another struct. + +- C++ destructors that invoke Python functions can be particularly troublesome as + destructors can sometimes get invoked in weird and unexpected circumstances as a result + of exceptions. + +- You should try running your code in a debug build. That will enable additional assertions + within pybind11 that will throw exceptions on certain GIL handling errors + (reference counting operations). + Binding sequence data types, iterators, the slicing protocol, etc. ================================================================== @@ -298,6 +352,15 @@ The class ``options`` allows you to selectively suppress auto-generated signatur m.def("add", [](int a, int b) { return a + b; }, "A function which adds two numbers"); } +pybind11 also appends all members of an enum to the resulting enum docstring. +This default behavior can be disabled by using the ``disable_enum_members_docstring()`` +function of the ``options`` class. + +With ``disable_user_defined_docstrings()`` all user defined docstrings of +``module_::def()``, ``class_::def()`` and ``enum_()`` are disabled, but the +function signatures and enum members are included in the docstring, unless they +are disabled separately. + Note that changes to the settings affect only function bindings created during the lifetime of the ``options`` instance. When it goes out of scope at the end of the module's init function, the default settings are restored to prevent unwanted side effects. diff --git a/docs/advanced/pycpp/numpy.rst b/docs/advanced/pycpp/numpy.rst index 0a81aa8f..07c96930 100644 --- a/docs/advanced/pycpp/numpy.rst +++ b/docs/advanced/pycpp/numpy.rst @@ -87,7 +87,7 @@ buffer objects (e.g. a NumPy matrix). /* Request a buffer descriptor from Python */ py::buffer_info info = b.request(); - /* Some sanity checks ... */ + /* Some basic validation checks ... */ if (info.format != py::format_descriptor<Scalar>::format()) throw std::runtime_error("Incompatible format: expected a double array!"); @@ -171,6 +171,31 @@ template parameter, and it ensures that non-conforming arguments are converted into an array satisfying the specified requirements instead of trying the next function overload. +There are several methods on arrays; the methods listed below under references +work, as well as the following functions based on the NumPy API: + +- ``.dtype()`` returns the type of the contained values. + +- ``.strides()`` returns a pointer to the strides of the array (optionally pass + an integer axis to get a number). + +- ``.flags()`` returns the flag settings. ``.writable()`` and ``.owndata()`` + are directly available. + +- ``.offset_at()`` returns the offset (optionally pass indices). + +- ``.squeeze()`` returns a view with length-1 axes removed. + +- ``.view(dtype)`` returns a view of the array with a different dtype. + +- ``.reshape({i, j, ...})`` returns a view of the array with a different shape. + ``.resize({...})`` is also available. + +- ``.index_at(i, j, ...)`` gets the count from the beginning to a given index. + + +There are also several methods for getting references (described below). + Structured types ================ @@ -233,8 +258,8 @@ by the compiler. The result is returned as a NumPy array of type .. code-block:: pycon - >>> x = np.array([[1, 3],[5, 7]]) - >>> y = np.array([[2, 4],[6, 8]]) + >>> x = np.array([[1, 3], [5, 7]]) + >>> y = np.array([[2, 4], [6, 8]]) >>> z = 3 >>> result = vectorized_func(x, y, z) @@ -345,21 +370,21 @@ The returned proxy object supports some of the same methods as ``py::array`` so that it can be used as a drop-in replacement for some existing, index-checked uses of ``py::array``: -- ``r.ndim()`` returns the number of dimensions +- ``.ndim()`` returns the number of dimensions -- ``r.data(1, 2, ...)`` and ``r.mutable_data(1, 2, ...)``` returns a pointer to +- ``.data(1, 2, ...)`` and ``r.mutable_data(1, 2, ...)``` returns a pointer to the ``const T`` or ``T`` data, respectively, at the given indices. The latter is only available to proxies obtained via ``a.mutable_unchecked()``. -- ``itemsize()`` returns the size of an item in bytes, i.e. ``sizeof(T)``. +- ``.itemsize()`` returns the size of an item in bytes, i.e. ``sizeof(T)``. -- ``ndim()`` returns the number of dimensions. +- ``.ndim()`` returns the number of dimensions. -- ``shape(n)`` returns the size of dimension ``n`` +- ``.shape(n)`` returns the size of dimension ``n`` -- ``size()`` returns the total number of elements (i.e. the product of the shapes). +- ``.size()`` returns the total number of elements (i.e. the product of the shapes). -- ``nbytes()`` returns the number of bytes used by the referenced elements +- ``.nbytes()`` returns the number of bytes used by the referenced elements (i.e. ``itemsize()`` times ``size()``). .. seealso:: @@ -370,15 +395,13 @@ uses of ``py::array``: Ellipsis ======== -Python 3 provides a convenient ``...`` ellipsis notation that is often used to +Python provides a convenient ``...`` ellipsis notation that is often used to slice multidimensional arrays. For instance, the following snippet extracts the middle dimensions of a tensor with the first and last index set to zero. -In Python 2, the syntactic sugar ``...`` is not available, but the singleton -``Ellipsis`` (of type ``ellipsis``) can still be used directly. .. code-block:: python - a = # a NumPy array + a = ... # a NumPy array b = a[0, ..., 0] The function ``py::ellipsis()`` function can be used to perform the same @@ -389,8 +412,6 @@ operation on the C++ side: py::array a = /* A NumPy array */; py::array b = a[py::make_tuple(0, py::ellipsis(), 0)]; -.. versionchanged:: 2.6 - ``py::ellipsis()`` is now also avaliable in Python 2. Memory view =========== @@ -412,7 +433,7 @@ following: { 2, 4 }, // shape (rows, cols) { sizeof(uint8_t) * 4, sizeof(uint8_t) } // strides in bytes ); - }) + }); This approach is meant for providing a ``memoryview`` for a C/C++ buffer not managed by Python. The user is responsible for managing the lifetime of the @@ -428,11 +449,7 @@ We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer: buffer, // buffer pointer sizeof(uint8_t) * 8 // buffer size ); - }) - -.. note:: - - ``memoryview::from_memory`` is not available in Python 2. + }); .. versionchanged:: 2.6 ``memoryview::from_memory`` added. diff --git a/docs/advanced/pycpp/object.rst b/docs/advanced/pycpp/object.rst index 6c7525ce..93e1a94d 100644 --- a/docs/advanced/pycpp/object.rst +++ b/docs/advanced/pycpp/object.rst @@ -20,6 +20,40 @@ Available types include :class:`handle`, :class:`object`, :class:`bool_`, Be sure to review the :ref:`pytypes_gotchas` before using this heavily in your C++ API. +.. _instantiating_compound_types: + +Instantiating compound Python types from C++ +============================================ + +Dictionaries can be initialized in the :class:`dict` constructor: + +.. code-block:: cpp + + using namespace pybind11::literals; // to bring in the `_a` literal + py::dict d("spam"_a=py::none(), "eggs"_a=42); + +A tuple of python objects can be instantiated using :func:`py::make_tuple`: + +.. code-block:: cpp + + py::tuple tup = py::make_tuple(42, py::none(), "spam"); + +Each element is converted to a supported Python type. + +A `simple namespace`_ can be instantiated using + +.. code-block:: cpp + + using namespace pybind11::literals; // to bring in the `_a` literal + py::object SimpleNamespace = py::module_::import("types").attr("SimpleNamespace"); + py::object ns = SimpleNamespace("spam"_a=py::none(), "eggs"_a=42); + +Attributes on a namespace can be modified with the :func:`py::delattr`, +:func:`py::getattr`, and :func:`py::setattr` functions. Simple namespaces can +be useful as lightweight stand-ins for class instances. + +.. _simple namespace: https://docs.python.org/3/library/types.html#types.SimpleNamespace + .. _casting_back_and_forth: Casting back and forth @@ -30,7 +64,7 @@ types to Python, which can be done using :func:`py::cast`: .. code-block:: cpp - MyClass *cls = ..; + MyClass *cls = ...; py::object obj = py::cast(cls); The reverse direction uses the following syntax: @@ -132,6 +166,7 @@ Keyword arguments are also supported. In Python, there is the usual call syntax: def f(number, say, to): ... # function code + f(1234, say="hello", to=some_instance) # keyword call in Python In C++, the same call can be made using: diff --git a/docs/advanced/pycpp/utilities.rst b/docs/advanced/pycpp/utilities.rst index c15051fb..af0f9cb2 100644 --- a/docs/advanced/pycpp/utilities.rst +++ b/docs/advanced/pycpp/utilities.rst @@ -28,7 +28,7 @@ Capturing standard output from ostream Often, a library will use the streams ``std::cout`` and ``std::cerr`` to print, but this does not play well with Python's standard ``sys.stdout`` and ``sys.stderr`` -redirection. Replacing a library's printing with `py::print <print>` may not +redirection. Replacing a library's printing with ``py::print <print>`` may not be feasible. This can be fixed using a guard around the library function that redirects output to the corresponding Python streams: @@ -47,15 +47,26 @@ redirects output to the corresponding Python streams: call_noisy_func(); }); +.. warning:: + + The implementation in ``pybind11/iostream.h`` is NOT thread safe. Multiple + threads writing to a redirected ostream concurrently cause data races + and potentially buffer overflows. Therefore it is currently a requirement + that all (possibly) concurrent redirected ostream writes are protected by + a mutex. #HelpAppreciated: Work on iostream.h thread safety. For more + background see the discussions under + `PR #2982 <https://github.com/pybind/pybind11/pull/2982>`_ and + `PR #2995 <https://github.com/pybind/pybind11/pull/2995>`_. + This method respects flushes on the output streams and will flush if needed when the scoped guard is destroyed. This allows the output to be redirected in real time, such as to a Jupyter notebook. The two arguments, the C++ stream and the Python output, are optional, and default to standard output if not given. An -extra type, `py::scoped_estream_redirect <scoped_estream_redirect>`, is identical +extra type, ``py::scoped_estream_redirect <scoped_estream_redirect>``, is identical except for defaulting to ``std::cerr`` and ``sys.stderr``; this can be useful with -`py::call_guard`, which allows multiple items, but uses the default constructor: +``py::call_guard``, which allows multiple items, but uses the default constructor: -.. code-block:: py +.. code-block:: cpp // Alternative: Call single function using call guard m.def("noisy_func", &call_noisy_function, @@ -63,7 +74,7 @@ except for defaulting to ``std::cerr`` and ``sys.stderr``; this can be useful wi py::scoped_estream_redirect>()); The redirection can also be done in Python with the addition of a context -manager, using the `py::add_ostream_redirect() <add_ostream_redirect>` function: +manager, using the ``py::add_ostream_redirect() <add_ostream_redirect>`` function: .. code-block:: cpp @@ -92,7 +103,7 @@ arguments to disable one of the streams if needed. Evaluating Python expressions from strings and files ==================================================== -pybind11 provides the `eval`, `exec` and `eval_file` functions to evaluate +pybind11 provides the ``eval``, ``exec`` and ``eval_file`` functions to evaluate Python expressions and statements. The following example illustrates how they can be used. diff --git a/docs/advanced/smart_ptrs.rst b/docs/advanced/smart_ptrs.rst index da57748c..3c40ce12 100644 --- a/docs/advanced/smart_ptrs.rst +++ b/docs/advanced/smart_ptrs.rst @@ -77,6 +77,7 @@ segmentation fault). .. code-block:: python from example import Parent + print(Parent().get_child()) The problem is that ``Parent::get_child()`` returns a pointer to an instance of @@ -156,7 +157,7 @@ specialized: PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr<T>); // Only needed if the type's `.get()` goes by another name - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <typename T> struct holder_helper<SmartPtr<T>> { // <-- specialization static const T *get(const SmartPtr<T> &p) { return p.getPointer(); } diff --git a/docs/basics.rst b/docs/basics.rst index 0b1d85cf..e9b24c7f 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -32,8 +32,7 @@ The last line will both compile and run the tests. Windows ------- -On Windows, only **Visual Studio 2015** and newer are supported since pybind11 relies -on various C++11 language features that break older versions of Visual Studio. +On Windows, only **Visual Studio 2017** and newer are supported. .. Note:: @@ -109,7 +108,7 @@ a file named :file:`example.cpp` with the following contents: PYBIND11_MODULE(example, m) { m.doc() = "pybind11 example plugin"; // optional module docstring - m.def("add", &add, "A function which adds two numbers"); + m.def("add", &add, "A function that adds two numbers"); } .. [#f1] In practice, implementation and binding code will generally be located @@ -166,12 +165,12 @@ load and execute the example: .. code-block:: pycon $ python - Python 2.7.10 (default, Aug 22 2015, 20:33:39) - [GCC 4.2.1 Compatible Apple LLVM 7.0.0 (clang-700.0.59.1)] on darwin + Python 3.9.10 (main, Jan 15 2022, 11:48:04) + [Clang 13.0.0 (clang-1300.0.29.3)] on darwin Type "help", "copyright", "credits" or "license" for more information. >>> import example >>> example.add(1, 2) - 3L + 3 >>> .. _keyword_args: diff --git a/docs/benchmark.py b/docs/benchmark.py index 33d78fb4..2150b6ca 100644 --- a/docs/benchmark.py +++ b/docs/benchmark.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- -import random -import os -import time import datetime as dt +import os +import random nfns = 4 # Functions per class nargs = 4 # Arguments per function @@ -13,20 +11,20 @@ def generate_dummy_code_pybind11(nclasses=10): bindings = "" for cl in range(nclasses): - decl += "class cl%03i;\n" % cl + decl += f"class cl{cl:03};\n" decl += "\n" for cl in range(nclasses): - decl += "class cl%03i {\n" % cl + decl += f"class {cl:03} {{\n" decl += "public:\n" - bindings += ' py::class_<cl%03i>(m, "cl%03i")\n' % (cl, cl) + bindings += f' py::class_<cl{cl:03}>(m, "cl{cl:03}")\n' for fn in range(nfns): ret = random.randint(0, nclasses - 1) params = [random.randint(0, nclasses - 1) for i in range(nargs)] - decl += " cl%03i *fn_%03i(" % (ret, fn) - decl += ", ".join("cl%03i *" % p for p in params) + decl += f" cl{ret:03} *fn_{fn:03}(" + decl += ", ".join(f"cl{p:03} *" for p in params) decl += ");\n" - bindings += ' .def("fn_%03i", &cl%03i::fn_%03i)\n' % (fn, cl, fn) + bindings += f' .def("fn_{fn:03}", &cl{cl:03}::fn_{fn:03})\n' decl += "};\n\n" bindings += " ;\n" @@ -44,23 +42,20 @@ def generate_dummy_code_boost(nclasses=10): bindings = "" for cl in range(nclasses): - decl += "class cl%03i;\n" % cl + decl += f"class cl{cl:03};\n" decl += "\n" for cl in range(nclasses): decl += "class cl%03i {\n" % cl decl += "public:\n" - bindings += ' py::class_<cl%03i>("cl%03i")\n' % (cl, cl) + bindings += f' py::class_<cl{cl:03}>("cl{cl:03}")\n' for fn in range(nfns): ret = random.randint(0, nclasses - 1) params = [random.randint(0, nclasses - 1) for i in range(nargs)] - decl += " cl%03i *fn_%03i(" % (ret, fn) - decl += ", ".join("cl%03i *" % p for p in params) + decl += f" cl{ret:03} *fn_{fn:03}(" + decl += ", ".join(f"cl{p:03} *" for p in params) decl += ");\n" - bindings += ( - ' .def("fn_%03i", &cl%03i::fn_%03i, py::return_value_policy<py::manage_new_object>())\n' - % (fn, cl, fn) - ) + bindings += f' .def("fn_{fn:03}", &cl{cl:03}::fn_{fn:03}, py::return_value_policy<py::manage_new_object>())\n' decl += "};\n\n" bindings += " ;\n" @@ -76,7 +71,7 @@ def generate_dummy_code_boost(nclasses=10): for codegen in [generate_dummy_code_pybind11, generate_dummy_code_boost]: print("{") for i in range(0, 10): - nclasses = 2 ** i + nclasses = 2**i with open("test.cpp", "w") as f: f.write(codegen(nclasses)) n1 = dt.datetime.now() diff --git a/docs/changelog.rst b/docs/changelog.rst index 157b2960..7bb115b9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,1146 @@ Changelog Starting with version 1.8.0, pybind11 releases use a `semantic versioning <http://semver.org>`_ policy. +Changes will be added here periodically from the "Suggested changelog entry" +block in pull request descriptions. + + +Version 2.11.0 (July 14, 2023) +----------------------------- + +New features: + +* The newly added ``pybind11::detail::is_move_constructible`` trait can be + specialized for cases in which ``std::is_move_constructible`` does not work + as needed. This is very similar to the long-established + ``pybind11::detail::is_copy_constructible``. + `#4631 <https://github.com/pybind/pybind11/pull/4631>`_ + +* Introduce ``recursive_container_traits``. + `#4623 <https://github.com/pybind/pybind11/pull/4623>`_ + +* ``pybind11/type_caster_pyobject_ptr.h`` was added to support automatic + wrapping of APIs that make use of ``PyObject *``. This header needs to + included explicitly (i.e. it is not included implicitly + with ``pybind/pybind11.h``). + `#4601 <https://github.com/pybind/pybind11/pull/4601>`_ + +* ``format_descriptor<>`` & ``npy_format_descriptor<>`` ``PyObject *`` + specializations were added. The latter enables ``py::array_t<PyObject *>`` + to/from-python conversions. + `#4674 <https://github.com/pybind/pybind11/pull/4674>`_ + +* ``buffer_info`` gained an ``item_type_is_equivalent_to<T>()`` member + function. + `#4674 <https://github.com/pybind/pybind11/pull/4674>`_ + +* The ``capsule`` API gained a user-friendly constructor + (``py::capsule(ptr, "name", dtor)``). + `#4720 <https://github.com/pybind/pybind11/pull/4720>`_ + +Changes: + +* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & + ``dec_ref()`` are now enabled by default again. + `#4246 <https://github.com/pybind/pybind11/pull/4246>`_ + +* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()`` + instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete + ``sys.path``. + `#4473 <https://github.com/pybind/pybind11/pull/4473>`_ + +* Cast errors now always include Python type information, even if + ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary + sizes slightly (~1.5%) but the error messages are much more informative. + `#4463 <https://github.com/pybind/pybind11/pull/4463>`_ + +* The docstring generation for the ``std::array``-list caster was fixed. + Previously, signatures included the size of the list in a non-standard, + non-spec compliant way. The new format conforms to PEP 593. + **Tooling for processing the docstrings may need to be updated accordingly.** + `#4679 <https://github.com/pybind/pybind11/pull/4679>`_ + +* Setter return values (which are inaccessible for all practical purposes) are + no longer converted to Python (only to be discarded). + `#4621 <https://github.com/pybind/pybind11/pull/4621>`_ + +* Allow lambda specified to function definition to be ``noexcept(true)`` + in C++17. + `#4593 <https://github.com/pybind/pybind11/pull/4593>`_ + +* Get rid of recursive template instantiations for concatenating type + signatures on C++17 and higher. + `#4587 <https://github.com/pybind/pybind11/pull/4587>`_ + +* Compatibility with Python 3.12 (beta). Note that the minimum pybind11 + ABI version for Python 3.12 is version 5. (The default ABI version + for Python versions up to and including 3.11 is still version 4.). + `#4570 <https://github.com/pybind/pybind11/pull/4570>`_ + +* With ``PYBIND11_INTERNALS_VERSION 5`` (default for Python 3.12+), MSVC builds + use ``std::hash<std::type_index>`` and ``std::equal_to<std::type_index>`` + instead of string-based type comparisons. This resolves issues when binding + types defined in the unnamed namespace. + `#4319 <https://github.com/pybind/pybind11/pull/4319>`_ + +* Python exception ``__notes__`` (introduced with Python 3.11) are now added to + the ``error_already_set::what()`` output. + `#4678 <https://github.com/pybind/pybind11/pull/4678>`_ + +Build system improvements: + +* CMake 3.27 support was added, CMake 3.4 support was dropped. + FindPython will be used if ``FindPythonInterp`` is not present. + `#4719 <https://github.com/pybind/pybind11/pull/4719>`_ + +* Update clang-tidy to 15 in CI. + `#4387 <https://github.com/pybind/pybind11/pull/4387>`_ + +* Moved the linting framework over to Ruff. + `#4483 <https://github.com/pybind/pybind11/pull/4483>`_ + +* Skip ``lto`` checks and target generation when + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. + `#4643 <https://github.com/pybind/pybind11/pull/4643>`_ + +* No longer inject ``-stdlib=libc++``, not needed for modern Pythons + (macOS 10.9+). + `#4639 <https://github.com/pybind/pybind11/pull/4639>`_ + +* PyPy 3.10 support was added, PyPy 3.7 support was dropped. + `#4728 <https://github.com/pybind/pybind11/pull/4728>`_ + +* Testing with Python 3.12 beta releases was added. + `#4713 <https://github.com/pybind/pybind11/pull/4713>`_ + + +Version 2.10.4 (Mar 16, 2023) +----------------------------- + +Changes: + +* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and + exits). + `#4526 <https://github.com/pybind/pybind11/pull/4526>`_ + +Bug Fixes: + +* Fix a warning when pydebug is enabled on Python 3.11. + `#4461 <https://github.com/pybind/pybind11/pull/4461>`_ + +* Ensure ``gil_scoped_release`` RAII is non-copyable. + `#4490 <https://github.com/pybind/pybind11/pull/4490>`_ + +* Ensure the tests dir does not show up with new versions of setuptools. + `#4510 <https://github.com/pybind/pybind11/pull/4510>`_ + +* Better stacklevel for a warning in setuptools helpers. + `#4516 <https://github.com/pybind/pybind11/pull/4516>`_ + +Version 2.10.3 (Jan 3, 2023) +---------------------------- + +Changes: + +* Temporarily made our GIL status assertions (added in 2.10.2) disabled by + default (re-enable manually by defining + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11). + `#4432 <https://github.com/pybind/pybind11/pull/4432>`_ + +* Improved error messages when ``inc_ref``/``dec_ref`` are called with an + invalid GIL state. + `#4427 <https://github.com/pybind/pybind11/pull/4427>`_ + `#4436 <https://github.com/pybind/pybind11/pull/4436>`_ + +Bug Fixes: + +* Some minor touchups found by static analyzers. + `#4440 <https://github.com/pybind/pybind11/pull/4440>`_ + + +Version 2.10.2 (Dec 20, 2022) +----------------------------- + +Changes: + +* ``scoped_interpreter`` constructor taking ``PyConfig``. + `#4330 <https://github.com/pybind/pybind11/pull/4330>`_ + +* ``pybind11/eigen/tensor.h`` adds converters to and from ``Eigen::Tensor`` and + ``Eigen::TensorMap``. + `#4201 <https://github.com/pybind/pybind11/pull/4201>`_ + +* ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` + ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if + ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11) + `#4246 <https://github.com/pybind/pybind11/pull/4246>`_ + +* Add option for enable/disable enum members in docstring. + `#2768 <https://github.com/pybind/pybind11/pull/2768>`_ + +* Fixed typing of ``KeysView``, ``ValuesView`` and ``ItemsView`` in ``bind_map``. + `#4353 <https://github.com/pybind/pybind11/pull/4353>`_ + +Bug fixes: + +* Bug fix affecting only Python 3.6 under very specific, uncommon conditions: + move ``PyEval_InitThreads()`` call to the correct location. + `#4350 <https://github.com/pybind/pybind11/pull/4350>`_ + +* Fix segfault bug when passing foreign native functions to functional.h. + `#4254 <https://github.com/pybind/pybind11/pull/4254>`_ + +Build system improvements: + +* Support setting PYTHON_LIBRARIES manually for Windows ARM cross-compilation + (classic mode). + `#4406 <https://github.com/pybind/pybind11/pull/4406>`_ + +* Extend IPO/LTO detection for ICX (a.k.a IntelLLVM) compiler. + `#4402 <https://github.com/pybind/pybind11/pull/4402>`_ + +* Allow calling ``find_package(pybind11 CONFIG)`` multiple times from separate + directories in the same CMake project and properly link Python (new mode). + `#4401 <https://github.com/pybind/pybind11/pull/4401>`_ + +* ``multiprocessing_set_spawn`` in pytest fixture for added safety. + `#4377 <https://github.com/pybind/pybind11/pull/4377>`_ + +* Fixed a bug in two pybind11/tools cmake scripts causing "Unknown arguments specified" errors. + `#4327 <https://github.com/pybind/pybind11/pull/4327>`_ + + + +Version 2.10.1 (Oct 31, 2022) +----------------------------- + +This is the first version to fully support embedding the newly released Python 3.11. + +Changes: + +* Allow ``pybind11::capsule`` constructor to take null destructor pointers. + `#4221 <https://github.com/pybind/pybind11/pull/4221>`_ + +* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11 + (established behavior). + `#4119 <https://github.com/pybind/pybind11/pull/4119>`_ + +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 <https://github.com/pybind/pybind11/pull/4216>`_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 <https://github.com/pybind/pybind11/pull/4298>`_ + +Bug fixes: + +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 <https://github.com/pybind/pybind11/pull/4294>`_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 <https://github.com/pybind/pybind11/pull/4234>`_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 <https://github.com/pybind/pybind11/pull/4275>`_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 <https://github.com/pybind/pybind11/pull/4278>`_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 <https://github.com/pybind/pybind11/pull/4297>`_ + +* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. + `#4188 <https://github.com/pybind/pybind11/pull/4188>`_ + +* Make augmented assignment operators non-const for the object-api. Behavior + was previously broken for augmented assignment operators. + `#4065 <https://github.com/pybind/pybind11/pull/4065>`_ + +* Add proper error checking to C++ bindings for Python list append and insert. + `#4208 <https://github.com/pybind/pybind11/pull/4208>`_ + +* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0. + `#4220 <https://github.com/pybind/pybind11/pull/4220>`_ + +* A workaround for PyPy was added in the ``py::error_already_set`` + implementation, related to PR `#1895 <https://github.com/pybind/pybind11/pull/1895>`_ + released with v2.10.0. + `#4079 <https://github.com/pybind/pybind11/pull/4079>`_ + +* Fixed compiler errors when C++23 ``std::forward_like`` is available. + `#4136 <https://github.com/pybind/pybind11/pull/4136>`_ + +* Properly raise exceptions in contains methods (like when an object in unhashable). + `#4209 <https://github.com/pybind/pybind11/pull/4209>`_ + +* Further improve another error in exception handling. + `#4232 <https://github.com/pybind/pybind11/pull/4232>`_ + +* ``get_local_internals()`` was made compatible with + ``finalize_interpreter()``, fixing potential freezes during interpreter + finalization. + `#4192 <https://github.com/pybind/pybind11/pull/4192>`_ + +Performance and style: + +* Reserve space in set and STL map casters if possible. This will prevent + unnecessary rehashing / resizing by knowing the number of keys ahead of time + for Python to C++ casting. This improvement will greatly speed up the casting + of large unordered maps and sets. + `#4194 <https://github.com/pybind/pybind11/pull/4194>`_ + +* GIL RAII scopes are non-copyable to avoid potential bugs. + `#4183 <https://github.com/pybind/pybind11/pull/4183>`_ + +* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT`` + macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in + macros as well. + `#4017 <https://github.com/pybind/pybind11/pull/4017>`_ + +* Optimize iterator advancement in C++ bindings. + `#4237 <https://github.com/pybind/pybind11/pull/4237>`_ + +* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict`` + for handling dynamic attribute dictionaries. + `#4106 <https://github.com/pybind/pybind11/pull/4106>`_ + +* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when + opening namespaces. Using namespace declarations and namespace qualification + remain the same as ``pybind11``. This is done to ensure consistent symbol + visibility. + `#4098 <https://github.com/pybind/pybind11/pull/4098>`_ + +* Mark ``detail::forward_like`` as constexpr. + `#4147 <https://github.com/pybind/pybind11/pull/4147>`_ + +* Optimize unpacking_collector when processing ``arg_v`` arguments. + `#4219 <https://github.com/pybind/pybind11/pull/4219>`_ + +* Optimize casting C++ object to ``None``. + `#4269 <https://github.com/pybind/pybind11/pull/4269>`_ + + +Build system improvements: + +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 <https://github.com/pybind/pybind11/pull/4195>`_ + +* Include a pkg-config file when installing pybind11, such as in the Python + package. + `#4077 <https://github.com/pybind/pybind11/pull/4077>`_ + +* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG`` + instead of ``Debug``. + `#4078 <https://github.com/pybind/pybind11/pull/4078>`_ + +* Followup to `#3948 <https://github.com/pybind/pybind11/pull/3948>`_, fixing vcpkg again. + `#4123 <https://github.com/pybind/pybind11/pull/4123>`_ + +Version 2.10.0 (Jul 15, 2022) +----------------------------- + +Removed support for Python 2.7, Python 3.5, and MSVC 2015. Support for MSVC +2017 is limited due to availability of CI runners; we highly recommend MSVC +2019 or 2022 be used. Initial support added for Python 3.11. + +New features: + +* ``py::anyset`` & ``py::frozenset`` were added, with copying (cast) to + ``std::set`` (similar to ``set``). + `#3901 <https://github.com/pybind/pybind11/pull/3901>`_ + +* Support bytearray casting to string. + `#3707 <https://github.com/pybind/pybind11/pull/3707>`_ + +* ``type_caster<std::monostate>`` was added. ``std::monostate`` is a tag type + that allows ``std::variant`` to act as an optional, or allows default + construction of a ``std::variant`` holding a non-default constructible type. + `#3818 <https://github.com/pybind/pybind11/pull/3818>`_ + +* ``pybind11::capsule::set_name`` added to mutate the name of the capsule instance. + `#3866 <https://github.com/pybind/pybind11/pull/3866>`_ + +* NumPy: dtype constructor from type number added, accessors corresponding to + Python API ``dtype.num``, ``dtype.byteorder``, ``dtype.flags`` and + ``dtype.alignment`` added. + `#3868 <https://github.com/pybind/pybind11/pull/3868>`_ + + +Changes: + +* Python 3.6 is now the minimum supported version. + `#3688 <https://github.com/pybind/pybind11/pull/3688>`_ + `#3719 <https://github.com/pybind/pybind11/pull/3719>`_ + +* The minimum version for MSVC is now 2017. + `#3722 <https://github.com/pybind/pybind11/pull/3722>`_ + +* Fix issues with CPython 3.11 betas and add to supported test matrix. + `#3923 <https://github.com/pybind/pybind11/pull/3923>`_ + +* ``error_already_set`` is now safer and more performant, especially for + exceptions with long tracebacks, by delaying computation. + `#1895 <https://github.com/pybind/pybind11/pull/1895>`_ + +* Improve exception handling in python ``str`` bindings. + `#3826 <https://github.com/pybind/pybind11/pull/3826>`_ + +* The bindings for capsules now have more consistent exception handling. + `#3825 <https://github.com/pybind/pybind11/pull/3825>`_ + +* ``PYBIND11_OBJECT_CVT`` and ``PYBIND11_OBJECT_CVT_DEFAULT`` macro can now be + used to define classes in namespaces other than pybind11. + `#3797 <https://github.com/pybind/pybind11/pull/3797>`_ + +* Error printing code now uses ``PYBIND11_DETAILED_ERROR_MESSAGES`` instead of + requiring ``NDEBUG``, allowing use with release builds if desired. + `#3913 <https://github.com/pybind/pybind11/pull/3913>`_ + +* Implicit conversion of the literal ``0`` to ``pybind11::handle`` is now disabled. + `#4008 <https://github.com/pybind/pybind11/pull/4008>`_ + + +Bug fixes: + +* Fix exception handling when ``pybind11::weakref()`` fails. + `#3739 <https://github.com/pybind/pybind11/pull/3739>`_ + +* ``module_::def_submodule`` was missing proper error handling. This is fixed now. + `#3973 <https://github.com/pybind/pybind11/pull/3973>`_ + +* The behavior or ``error_already_set`` was made safer and the highly opaque + "Unknown internal error occurred" message was replaced with a more helpful + message. + `#3982 <https://github.com/pybind/pybind11/pull/3982>`_ + +* ``error_already_set::what()`` now handles non-normalized exceptions correctly. + `#3971 <https://github.com/pybind/pybind11/pull/3971>`_ + +* Support older C++ compilers where filesystem is not yet part of the standard + library and is instead included in ``std::experimental::filesystem``. + `#3840 <https://github.com/pybind/pybind11/pull/3840>`_ + +* Fix ``-Wfree-nonheap-object`` warnings produced by GCC by avoiding returning + pointers to static objects with ``return_value_policy::take_ownership``. + `#3946 <https://github.com/pybind/pybind11/pull/3946>`_ + +* Fix cast from pytype rvalue to another pytype. + `#3949 <https://github.com/pybind/pybind11/pull/3949>`_ + +* Ensure proper behavior when garbage collecting classes with dynamic attributes in Python >=3.9. + `#4051 <https://github.com/pybind/pybind11/pull/4051>`_ + +* A couple long-standing ``PYBIND11_NAMESPACE`` + ``__attribute__((visibility("hidden")))`` inconsistencies are now fixed + (affects only unusual environments). + `#4043 <https://github.com/pybind/pybind11/pull/4043>`_ + +* ``pybind11::detail::get_internals()`` is now resilient to in-flight Python + exceptions. + `#3981 <https://github.com/pybind/pybind11/pull/3981>`_ + +* Arrays with a dimension of size 0 are now properly converted to dynamic Eigen + matrices (more common in NumPy 1.23). + `#4038 <https://github.com/pybind/pybind11/pull/4038>`_ + +* Avoid catching unrelated errors when importing NumPy. + `#3974 <https://github.com/pybind/pybind11/pull/3974>`_ + +Performance and style: + +* Added an accessor overload of ``(object &&key)`` to reference steal the + object when using python types as keys. This prevents unnecessary reference + count overhead for attr, dictionary, tuple, and sequence look ups. Added + additional regression tests. Fixed a performance bug the caused accessor + assignments to potentially perform unnecessary copies. + `#3970 <https://github.com/pybind/pybind11/pull/3970>`_ + +* Perfect forward all args of ``make_iterator``. + `#3980 <https://github.com/pybind/pybind11/pull/3980>`_ + +* Avoid potential bug in pycapsule destructor by adding an ``error_guard`` to + one of the dtors. + `#3958 <https://github.com/pybind/pybind11/pull/3958>`_ + +* Optimize dictionary access in ``strip_padding`` for numpy. + `#3994 <https://github.com/pybind/pybind11/pull/3994>`_ + +* ``stl_bind.h`` bindings now take slice args as a const-ref. + `#3852 <https://github.com/pybind/pybind11/pull/3852>`_ + +* Made slice constructor more consistent, and improve performance of some + casters by allowing reference stealing. + `#3845 <https://github.com/pybind/pybind11/pull/3845>`_ + +* Change numpy dtype from_args method to use const ref. + `#3878 <https://github.com/pybind/pybind11/pull/3878>`_ + +* Follow rule of three to ensure ``PyErr_Restore`` is called only once. + `#3872 <https://github.com/pybind/pybind11/pull/3872>`_ + +* Added missing perfect forwarding for ``make_iterator`` functions. + `#3860 <https://github.com/pybind/pybind11/pull/3860>`_ + +* Optimize c++ to python function casting by using the rvalue caster. + `#3966 <https://github.com/pybind/pybind11/pull/3966>`_ + +* Optimize Eigen sparse matrix casting by removing unnecessary temporary. + `#4064 <https://github.com/pybind/pybind11/pull/4064>`_ + +* Avoid potential implicit copy/assignment constructors causing double free in + ``strdup_gaurd``. + `#3905 <https://github.com/pybind/pybind11/pull/3905>`_ + +* Enable clang-tidy checks ``misc-definitions-in-headers``, + ``modernize-loop-convert``, and ``modernize-use-nullptr``. + `#3881 <https://github.com/pybind/pybind11/pull/3881>`_ + `#3988 <https://github.com/pybind/pybind11/pull/3988>`_ + + +Build system improvements: + +* CMake: Fix file extension on Windows with cp36 and cp37 using FindPython. + `#3919 <https://github.com/pybind/pybind11/pull/3919>`_ + +* CMake: Support multiple Python targets (such as on vcpkg). + `#3948 <https://github.com/pybind/pybind11/pull/3948>`_ + +* CMake: Fix issue with NVCC on Windows. + `#3947 <https://github.com/pybind/pybind11/pull/3947>`_ + +* CMake: Drop the bitness check on cross compiles (like targeting WebAssembly + via Emscripten). + `#3959 <https://github.com/pybind/pybind11/pull/3959>`_ + +* Add MSVC builds in debug mode to CI. + `#3784 <https://github.com/pybind/pybind11/pull/3784>`_ + +* MSVC 2022 C++20 coverage was added to GitHub Actions, including Eigen. + `#3732 <https://github.com/pybind/pybind11/pull/3732>`_, + `#3741 <https://github.com/pybind/pybind11/pull/3741>`_ + + +Backend and tidying up: + +* New theme for the documentation. + `#3109 <https://github.com/pybind/pybind11/pull/3109>`_ + +* Remove idioms in code comments. Use more inclusive language. + `#3809 <https://github.com/pybind/pybind11/pull/3809>`_ + +* ``#include <iostream>`` was removed from the ``pybind11/stl.h`` header. Your + project may break if it has a transitive dependency on this include. The fix + is to "Include What You Use". + `#3928 <https://github.com/pybind/pybind11/pull/3928>`_ + +* Avoid ``setup.py <command>`` usage in internal tests. + `#3734 <https://github.com/pybind/pybind11/pull/3734>`_ + + +Version 2.9.2 (Mar 29, 2022) +---------------------------- + +Changes: + +* Enum now has an ``__index__`` method on Python <3.8 too. + `#3700 <https://github.com/pybind/pybind11/pull/3700>`_ + +* Local internals are now cleared after finalizing the interpreter. + `#3744 <https://github.com/pybind/pybind11/pull/3744>`_ + +Bug fixes: + +* Better support for Python 3.11 alphas. + `#3694 <https://github.com/pybind/pybind11/pull/3694>`_ + +* ``PYBIND11_TYPE_CASTER`` now uses fully qualified symbols, so it can be used + outside of ``pybind11::detail``. + `#3758 <https://github.com/pybind/pybind11/pull/3758>`_ + +* Some fixes for PyPy 3.9. + `#3768 <https://github.com/pybind/pybind11/pull/3768>`_ + +* Fixed a potential memleak in PyPy in ``get_type_override``. + `#3774 <https://github.com/pybind/pybind11/pull/3774>`_ + +* Fix usage of ``VISIBILITY_INLINES_HIDDEN``. + `#3721 <https://github.com/pybind/pybind11/pull/3721>`_ + + +Build system improvements: + +* Uses ``sysconfig`` module to determine installation locations on Python >= + 3.10, instead of ``distutils`` which has been deprecated. + `#3764 <https://github.com/pybind/pybind11/pull/3764>`_ + +* Support Catch 2.13.5+ (supporting GLIBC 2.34+). + `#3679 <https://github.com/pybind/pybind11/pull/3679>`_ + +* Fix test failures with numpy 1.22 by ignoring whitespace when comparing + ``str()`` of dtypes. + `#3682 <https://github.com/pybind/pybind11/pull/3682>`_ + + +Backend and tidying up: + +* clang-tidy: added ``readability-qualified-auto``, + ``readability-braces-around-statements``, + ``cppcoreguidelines-prefer-member-initializer``, + ``clang-analyzer-optin.performance.Padding``, + ``cppcoreguidelines-pro-type-static-cast-downcast``, and + ``readability-inconsistent-declaration-parameter-name``. + `#3702 <https://github.com/pybind/pybind11/pull/3702>`_, + `#3699 <https://github.com/pybind/pybind11/pull/3699>`_, + `#3716 <https://github.com/pybind/pybind11/pull/3716>`_, + `#3709 <https://github.com/pybind/pybind11/pull/3709>`_ + +* clang-format was added to the pre-commit actions, and the entire code base + automatically reformatted (after several iterations preparing for this leap). + `#3713 <https://github.com/pybind/pybind11/pull/3713>`_ + + +Version 2.9.1 (Feb 2, 2022) +--------------------------- + +Changes: + +* If possible, attach Python exception with ``py::raise_from`` to ``TypeError`` + when casting from C++ to Python. This will give additional info if Python + exceptions occur in the caster. Adds a test case of trying to convert a set + from C++ to Python when the hash function is not defined in Python. + `#3605 <https://github.com/pybind/pybind11/pull/3605>`_ + +* Add a mapping of C++11 nested exceptions to their Python exception + equivalent using ``py::raise_from``. This attaches the nested exceptions in + Python using the ``__cause__`` field. + `#3608 <https://github.com/pybind/pybind11/pull/3608>`_ + +* Propagate Python exception traceback using ``raise_from`` if a pybind11 + function runs out of overloads. + `#3671 <https://github.com/pybind/pybind11/pull/3671>`_ + +* ``py::multiple_inheritance`` is now only needed when C++ bases are hidden + from pybind11. + `#3650 <https://github.com/pybind/pybind11/pull/3650>`_ and + `#3659 <https://github.com/pybind/pybind11/pull/3659>`_ + + +Bug fixes: + +* Remove a boolean cast in ``numpy.h`` that causes MSVC C4800 warnings when + compiling against Python 3.10 or newer. + `#3669 <https://github.com/pybind/pybind11/pull/3669>`_ + +* Render ``py::bool_`` and ``py::float_`` as ``bool`` and ``float`` + respectively. + `#3622 <https://github.com/pybind/pybind11/pull/3622>`_ + +Build system improvements: + +* Fix CMake extension suffix computation on Python 3.10+. + `#3663 <https://github.com/pybind/pybind11/pull/3663>`_ + +* Allow ``CMAKE_ARGS`` to override CMake args in pybind11's own ``setup.py``. + `#3577 <https://github.com/pybind/pybind11/pull/3577>`_ + +* Remove a few deprecated c-headers. + `#3610 <https://github.com/pybind/pybind11/pull/3610>`_ + +* More uniform handling of test targets. + `#3590 <https://github.com/pybind/pybind11/pull/3590>`_ + +* Add clang-tidy readability check to catch potentially swapped function args. + `#3611 <https://github.com/pybind/pybind11/pull/3611>`_ + + +Version 2.9.0 (Dec 28, 2021) +---------------------------- + +This is the last version to support Python 2.7 and 3.5. + +New Features: + +* Allow ``py::args`` to be followed by other arguments; the remaining arguments + are implicitly keyword-only, as if a ``py::kw_only{}`` annotation had been + used. + `#3402 <https://github.com/pybind/pybind11/pull/3402>`_ + +Changes: + +* Make str/bytes/memoryview more interoperable with ``std::string_view``. + `#3521 <https://github.com/pybind/pybind11/pull/3521>`_ + +* Replace ``_`` with ``const_name`` in internals, avoid defining ``pybind::_`` + if ``_`` defined as macro (common gettext usage) + `#3423 <https://github.com/pybind/pybind11/pull/3423>`_ + + +Bug fixes: + +* Fix a rare warning about extra copy in an Eigen constructor. + `#3486 <https://github.com/pybind/pybind11/pull/3486>`_ + +* Fix caching of the C++ overrides. + `#3465 <https://github.com/pybind/pybind11/pull/3465>`_ + +* Add missing ``std::forward`` calls to some ``cpp_function`` overloads. + `#3443 <https://github.com/pybind/pybind11/pull/3443>`_ + +* Support PyPy 7.3.7 and the PyPy3.8 beta. Test python-3.11 on PRs with the + ``python dev`` label. + `#3419 <https://github.com/pybind/pybind11/pull/3419>`_ + +* Replace usage of deprecated ``Eigen::MappedSparseMatrix`` with + ``Eigen::Map<Eigen::SparseMatrix<...>>`` for Eigen 3.3+. + `#3499 <https://github.com/pybind/pybind11/pull/3499>`_ + +* Tweaks to support Microsoft Visual Studio 2022. + `#3497 <https://github.com/pybind/pybind11/pull/3497>`_ + +Build system improvements: + +* Nicer CMake printout and IDE organisation for pybind11's own tests. + `#3479 <https://github.com/pybind/pybind11/pull/3479>`_ + +* CMake: report version type as part of the version string to avoid a spurious + space in the package status message. + `#3472 <https://github.com/pybind/pybind11/pull/3472>`_ + +* Flags starting with ``-g`` in ``$CFLAGS`` and ``$CPPFLAGS`` are no longer + overridden by ``.Pybind11Extension``. + `#3436 <https://github.com/pybind/pybind11/pull/3436>`_ + +* Ensure ThreadPool is closed in ``setup_helpers``. + `#3548 <https://github.com/pybind/pybind11/pull/3548>`_ + +* Avoid LTS on ``mips64`` and ``ppc64le`` (reported broken). + `#3557 <https://github.com/pybind/pybind11/pull/3557>`_ + + +v2.8.1 (Oct 27, 2021) +--------------------- + +Changes and additions: + +* The simple namespace creation shortcut added in 2.8.0 was deprecated due to + usage of CPython internal API, and will be removed soon. Use + ``py::module_::import("types").attr("SimpleNamespace")``. + `#3374 <https://github.com/pybinyyd/pybind11/pull/3374>`_ + +* Add C++ Exception type to throw and catch ``AttributeError``. Useful for + defining custom ``__setattr__`` and ``__getattr__`` methods. + `#3387 <https://github.com/pybind/pybind11/pull/3387>`_ + +Fixes: + +* Fixed the potential for dangling references when using properties with + ``std::optional`` types. + `#3376 <https://github.com/pybind/pybind11/pull/3376>`_ + +* Modernize usage of ``PyCodeObject`` on Python 3.9+ (moving toward support for + Python 3.11a1) + `#3368 <https://github.com/pybind/pybind11/pull/3368>`_ + +* A long-standing bug in ``eigen.h`` was fixed (originally PR #3343). The bug + was unmasked by newly added ``static_assert``'s in the Eigen 3.4.0 release. + `#3352 <https://github.com/pybind/pybind11/pull/3352>`_ + +* Support multiple raw inclusion of CMake helper files (Conan.io does this for + multi-config generators). + `#3420 <https://github.com/pybind/pybind11/pull/3420>`_ + +* Fix harmless warning on upcoming CMake 3.22. + `#3368 <https://github.com/pybind/pybind11/pull/3368>`_ + +* Fix 2.8.0 regression with MSVC 2017 + C++17 mode + Python 3. + `#3407 <https://github.com/pybind/pybind11/pull/3407>`_ + +* Fix 2.8.0 regression that caused undefined behavior (typically + segfaults) in ``make_key_iterator``/``make_value_iterator`` if dereferencing + the iterator returned a temporary value instead of a reference. + `#3348 <https://github.com/pybind/pybind11/pull/3348>`_ + + +v2.8.0 (Oct 4, 2021) +-------------------- + +New features: + +* Added ``py::raise_from`` to enable chaining exceptions. + `#3215 <https://github.com/pybind/pybind11/pull/3215>`_ + +* Allow exception translators to be optionally registered local to a module + instead of applying globally across all pybind11 modules. Use + ``register_local_exception_translator(ExceptionTranslator&& translator)`` + instead of ``register_exception_translator(ExceptionTranslator&& + translator)`` to keep your exception remapping code local to the module. + `#2650 <https://github.com/pybinyyd/pybind11/pull/2650>`_ + +* Add ``make_simple_namespace`` function for instantiating Python + ``SimpleNamespace`` objects. **Deprecated in 2.8.1.** + `#2840 <https://github.com/pybind/pybind11/pull/2840>`_ + +* ``pybind11::scoped_interpreter`` and ``initialize_interpreter`` have new + arguments to allow ``sys.argv`` initialization. + `#2341 <https://github.com/pybind/pybind11/pull/2341>`_ + +* Allow Python builtins to be used as callbacks in CPython. + `#1413 <https://github.com/pybind/pybind11/pull/1413>`_ + +* Added ``view`` to view arrays with a different datatype. + `#987 <https://github.com/pybind/pybind11/pull/987>`_ + +* Implemented ``reshape`` on arrays. + `#984 <https://github.com/pybind/pybind11/pull/984>`_ + +* Enable defining custom ``__new__`` methods on classes by fixing bug + preventing overriding methods if they have non-pybind11 siblings. + `#3265 <https://github.com/pybind/pybind11/pull/3265>`_ + +* Add ``make_value_iterator()``, and fix ``make_key_iterator()`` to return + references instead of copies. + `#3293 <https://github.com/pybind/pybind11/pull/3293>`_ + +* Improve the classes generated by ``bind_map``: `#3310 <https://github.com/pybind/pybind11/pull/3310>`_ + + * Change ``.items`` from an iterator to a dictionary view. + * Add ``.keys`` and ``.values`` (both dictionary views). + * Allow ``__contains__`` to take any object. + +* ``pybind11::custom_type_setup`` was added, for customizing the + ``PyHeapTypeObject`` corresponding to a class, which may be useful for + enabling garbage collection support, among other things. + `#3287 <https://github.com/pybind/pybind11/pull/3287>`_ + + +Changes: + +* Set ``__file__`` constant when running ``eval_file`` in an embedded interpreter. + `#3233 <https://github.com/pybind/pybind11/pull/3233>`_ + +* Python objects and (C++17) ``std::optional`` now accepted in ``py::slice`` + constructor. + `#1101 <https://github.com/pybind/pybind11/pull/1101>`_ + +* The pybind11 proxy types ``str``, ``bytes``, ``bytearray``, ``tuple``, + ``list`` now consistently support passing ``ssize_t`` values for sizes and + indexes. Previously, only ``size_t`` was accepted in several interfaces. + `#3219 <https://github.com/pybind/pybind11/pull/3219>`_ + +* Avoid evaluating ``PYBIND11_TLS_REPLACE_VALUE`` arguments more than once. + `#3290 <https://github.com/pybind/pybind11/pull/3290>`_ + +Fixes: + +* Bug fix: enum value's ``__int__`` returning non-int when underlying type is + bool or of char type. + `#1334 <https://github.com/pybind/pybind11/pull/1334>`_ + +* Fixes bug in setting error state in Capsule's pointer methods. + `#3261 <https://github.com/pybind/pybind11/pull/3261>`_ + +* A long-standing memory leak in ``py::cpp_function::initialize`` was fixed. + `#3229 <https://github.com/pybind/pybind11/pull/3229>`_ + +* Fixes thread safety for some ``pybind11::type_caster`` which require lifetime + extension, such as for ``std::string_view``. + `#3237 <https://github.com/pybind/pybind11/pull/3237>`_ + +* Restore compatibility with gcc 4.8.4 as distributed by ubuntu-trusty, linuxmint-17. + `#3270 <https://github.com/pybind/pybind11/pull/3270>`_ + + +Build system improvements: + +* Fix regression in CMake Python package config: improper use of absolute path. + `#3144 <https://github.com/pybind/pybind11/pull/3144>`_ + +* Cached Python version information could become stale when CMake was re-run + with a different Python version. The build system now detects this and + updates this information. + `#3299 <https://github.com/pybind/pybind11/pull/3299>`_ + +* Specified UTF8-encoding in setup.py calls of open(). + `#3137 <https://github.com/pybind/pybind11/pull/3137>`_ + +* Fix a harmless warning from CMake 3.21 with the classic Python discovery. + `#3220 <https://github.com/pybind/pybind11/pull/3220>`_ + +* Eigen repo and version can now be specified as cmake options. + `#3324 <https://github.com/pybind/pybind11/pull/3324>`_ + + +Backend and tidying up: + +* Reduced thread-local storage required for keeping alive temporary data for + type conversion to one key per ABI version, rather than one key per extension + module. This makes the total thread-local storage required by pybind11 2 + keys per ABI version. + `#3275 <https://github.com/pybind/pybind11/pull/3275>`_ + +* Optimize NumPy array construction with additional moves. + `#3183 <https://github.com/pybind/pybind11/pull/3183>`_ + +* Conversion to ``std::string`` and ``std::string_view`` now avoids making an + extra copy of the data on Python >= 3.3. + `#3257 <https://github.com/pybind/pybind11/pull/3257>`_ + +* Remove const modifier from certain C++ methods on Python collections + (``list``, ``set``, ``dict``) such as (``clear()``, ``append()``, + ``insert()``, etc...) and annotated them with ``py-non-const``. + +* Enable readability ``clang-tidy-const-return`` and remove useless consts. + `#3254 <https://github.com/pybind/pybind11/pull/3254>`_ + `#3194 <https://github.com/pybind/pybind11/pull/3194>`_ + +* The clang-tidy ``google-explicit-constructor`` option was enabled. + `#3250 <https://github.com/pybind/pybind11/pull/3250>`_ + +* Mark a pytype move constructor as noexcept (perf). + `#3236 <https://github.com/pybind/pybind11/pull/3236>`_ + +* Enable clang-tidy check to guard against inheritance slicing. + `#3210 <https://github.com/pybind/pybind11/pull/3210>`_ + +* Legacy warning suppression pragma were removed from eigen.h. On Unix + platforms, please use -isystem for Eigen include directories, to suppress + compiler warnings originating from Eigen headers. Note that CMake does this + by default. No adjustments are needed for Windows. + `#3198 <https://github.com/pybind/pybind11/pull/3198>`_ + +* Format pybind11 with isort consistent ordering of imports + `#3195 <https://github.com/pybind/pybind11/pull/3195>`_ + +* The warnings-suppression "pragma clamp" at the top/bottom of pybind11 was + removed, clearing the path to refactoring and IWYU cleanup. + `#3186 <https://github.com/pybind/pybind11/pull/3186>`_ + +* Enable most bugprone checks in clang-tidy and fix the found potential bugs + and poor coding styles. + `#3166 <https://github.com/pybind/pybind11/pull/3166>`_ + +* Add ``clang-tidy-readability`` rules to make boolean casts explicit improving + code readability. Also enabled other misc and readability clang-tidy checks. + `#3148 <https://github.com/pybind/pybind11/pull/3148>`_ + +* Move object in ``.pop()`` for list. + `#3116 <https://github.com/pybind/pybind11/pull/3116>`_ + + + + +v2.7.1 (Aug 3, 2021) +--------------------- + +Minor missing functionality added: + +* Allow Python builtins to be used as callbacks in CPython. + `#1413 <https://github.com/pybind/pybind11/pull/1413>`_ + +Bug fixes: + +* Fix regression in CMake Python package config: improper use of absolute path. + `#3144 <https://github.com/pybind/pybind11/pull/3144>`_ + +* Fix Mingw64 and add to the CI testing matrix. + `#3132 <https://github.com/pybind/pybind11/pull/3132>`_ + +* Specified UTF8-encoding in setup.py calls of open(). + `#3137 <https://github.com/pybind/pybind11/pull/3137>`_ + +* Add clang-tidy-readability rules to make boolean casts explicit improving + code readability. Also enabled other misc and readability clang-tidy checks. + `#3148 <https://github.com/pybind/pybind11/pull/3148>`_ + +* Move object in ``.pop()`` for list. + `#3116 <https://github.com/pybind/pybind11/pull/3116>`_ + +Backend and tidying up: + +* Removed and fixed warning suppressions. + `#3127 <https://github.com/pybind/pybind11/pull/3127>`_ + `#3129 <https://github.com/pybind/pybind11/pull/3129>`_ + `#3135 <https://github.com/pybind/pybind11/pull/3135>`_ + `#3141 <https://github.com/pybind/pybind11/pull/3141>`_ + `#3142 <https://github.com/pybind/pybind11/pull/3142>`_ + `#3150 <https://github.com/pybind/pybind11/pull/3150>`_ + `#3152 <https://github.com/pybind/pybind11/pull/3152>`_ + `#3160 <https://github.com/pybind/pybind11/pull/3160>`_ + `#3161 <https://github.com/pybind/pybind11/pull/3161>`_ + + +v2.7.0 (Jul 16, 2021) +--------------------- + +New features: + +* Enable ``py::implicitly_convertible<py::none, ...>`` for + ``py::class_``-wrapped types. + `#3059 <https://github.com/pybind/pybind11/pull/3059>`_ + +* Allow function pointer extraction from overloaded functions. + `#2944 <https://github.com/pybind/pybind11/pull/2944>`_ + +* NumPy: added ``.char_()`` to type which gives the NumPy public ``char`` + result, which also distinguishes types by bit length (unlike ``.kind()``). + `#2864 <https://github.com/pybind/pybind11/pull/2864>`_ + +* Add ``pybind11::bytearray`` to manipulate ``bytearray`` similar to ``bytes``. + `#2799 <https://github.com/pybind/pybind11/pull/2799>`_ + +* ``pybind11/stl/filesystem.h`` registers a type caster that, on C++17/Python + 3.6+, converts ``std::filesystem::path`` to ``pathlib.Path`` and any + ``os.PathLike`` to ``std::filesystem::path``. + `#2730 <https://github.com/pybind/pybind11/pull/2730>`_ + +* A ``PYBIND11_VERSION_HEX`` define was added, similar to ``PY_VERSION_HEX``. + `#3120 <https://github.com/pybind/pybind11/pull/3120>`_ + + + +Changes: + +* ``py::str`` changed to exclusively hold ``PyUnicodeObject``. Previously + ``py::str`` could also hold ``bytes``, which is probably surprising, was + never documented, and can mask bugs (e.g. accidental use of ``py::str`` + instead of ``py::bytes``). + `#2409 <https://github.com/pybind/pybind11/pull/2409>`_ + +* Add a safety guard to ensure that the Python GIL is held when C++ calls back + into Python via ``object_api<>::operator()`` (e.g. ``py::function`` + ``__call__``). (This feature is available for Python 3.6+ only.) + `#2919 <https://github.com/pybind/pybind11/pull/2919>`_ + +* Catch a missing ``self`` argument in calls to ``__init__()``. + `#2914 <https://github.com/pybind/pybind11/pull/2914>`_ + +* Use ``std::string_view`` if available to avoid a copy when passing an object + to a ``std::ostream``. + `#3042 <https://github.com/pybind/pybind11/pull/3042>`_ + +* An important warning about thread safety was added to the ``iostream.h`` + documentation; attempts to make ``py::scoped_ostream_redirect`` thread safe + have been removed, as it was only partially effective. + `#2995 <https://github.com/pybind/pybind11/pull/2995>`_ + + +Fixes: + +* Performance: avoid unnecessary strlen calls. + `#3058 <https://github.com/pybind/pybind11/pull/3058>`_ + +* Fix auto-generated documentation string when using ``const T`` in + ``pyarray_t``. + `#3020 <https://github.com/pybind/pybind11/pull/3020>`_ + +* Unify error messages thrown by ``simple_collector``/``unpacking_collector``. + `#3013 <https://github.com/pybind/pybind11/pull/3013>`_ + +* ``pybind11::builtin_exception`` is now explicitly exported, which means the + types included/defined in different modules are identical, and exceptions + raised in different modules can be caught correctly. The documentation was + updated to explain that custom exceptions that are used across module + boundaries need to be explicitly exported as well. + `#2999 <https://github.com/pybind/pybind11/pull/2999>`_ + +* Fixed exception when printing UTF-8 to a ``scoped_ostream_redirect``. + `#2982 <https://github.com/pybind/pybind11/pull/2982>`_ + +* Pickle support enhancement: ``setstate`` implementation will attempt to + ``setattr`` ``__dict__`` only if the unpickled ``dict`` object is not empty, + to not force use of ``py::dynamic_attr()`` unnecessarily. + `#2972 <https://github.com/pybind/pybind11/pull/2972>`_ + +* Allow negative timedelta values to roundtrip. + `#2870 <https://github.com/pybind/pybind11/pull/2870>`_ + +* Fix unchecked errors could potentially swallow signals/other exceptions. + `#2863 <https://github.com/pybind/pybind11/pull/2863>`_ + +* Add null pointer check with ``std::localtime``. + `#2846 <https://github.com/pybind/pybind11/pull/2846>`_ + +* Fix the ``weakref`` constructor from ``py::object`` to create a new + ``weakref`` on conversion. + `#2832 <https://github.com/pybind/pybind11/pull/2832>`_ + +* Avoid relying on exceptions in C++17 when getting a ``shared_ptr`` holder + from a ``shared_from_this`` class. + `#2819 <https://github.com/pybind/pybind11/pull/2819>`_ + +* Allow the codec's exception to be raised instead of :code:`RuntimeError` when + casting from :code:`py::str` to :code:`std::string`. + `#2903 <https://github.com/pybind/pybind11/pull/2903>`_ + + +Build system improvements: + +* In ``setup_helpers.py``, test for platforms that have some multiprocessing + features but lack semaphores, which ``ParallelCompile`` requires. + `#3043 <https://github.com/pybind/pybind11/pull/3043>`_ + +* Fix ``pybind11_INCLUDE_DIR`` in case ``CMAKE_INSTALL_INCLUDEDIR`` is + absolute. + `#3005 <https://github.com/pybind/pybind11/pull/3005>`_ + +* Fix bug not respecting ``WITH_SOABI`` or ``WITHOUT_SOABI`` to CMake. + `#2938 <https://github.com/pybind/pybind11/pull/2938>`_ + +* Fix the default ``Pybind11Extension`` compilation flags with a Mingw64 python. + `#2921 <https://github.com/pybind/pybind11/pull/2921>`_ + +* Clang on Windows: do not pass ``/MP`` (ignored flag). + `#2824 <https://github.com/pybind/pybind11/pull/2824>`_ + +* ``pybind11.setup_helpers.intree_extensions`` can be used to generate + ``Pybind11Extension`` instances from cpp files placed in the Python package + source tree. + `#2831 <https://github.com/pybind/pybind11/pull/2831>`_ + +Backend and tidying up: + +* Enable clang-tidy performance, readability, and modernization checks + throughout the codebase to enforce best coding practices. + `#3046 <https://github.com/pybind/pybind11/pull/3046>`_, + `#3049 <https://github.com/pybind/pybind11/pull/3049>`_, + `#3051 <https://github.com/pybind/pybind11/pull/3051>`_, + `#3052 <https://github.com/pybind/pybind11/pull/3052>`_, + `#3080 <https://github.com/pybind/pybind11/pull/3080>`_, and + `#3094 <https://github.com/pybind/pybind11/pull/3094>`_ + + +* Checks for common misspellings were added to the pre-commit hooks. + `#3076 <https://github.com/pybind/pybind11/pull/3076>`_ + +* Changed ``Werror`` to stricter ``Werror-all`` for Intel compiler and fixed + minor issues. + `#2948 <https://github.com/pybind/pybind11/pull/2948>`_ + +* Fixed compilation with GCC < 5 when the user defines ``_GLIBCXX_USE_CXX11_ABI``. + `#2956 <https://github.com/pybind/pybind11/pull/2956>`_ + +* Added nox support for easier local testing and linting of contributions. + `#3101 <https://github.com/pybind/pybind11/pull/3101>`_ and + `#3121 <https://github.com/pybind/pybind11/pull/3121>`_ + +* Avoid RTD style issue with docutils 0.17+. + `#3119 <https://github.com/pybind/pybind11/pull/3119>`_ + +* Support pipx run, such as ``pipx run pybind11 --include`` for a quick compile. + `#3117 <https://github.com/pybind/pybind11/pull/3117>`_ + + v2.6.2 (Jan 26, 2021) --------------------- @@ -86,7 +1226,7 @@ Bug fixes: * Fix ``py::gil_scoped_acquire`` assert with CPython 3.9 debug build. `#2683 <https://github.com/pybind/pybind11/pull/2683>`_ -* Fix issue with a test failing on PyTest 6.2. +* Fix issue with a test failing on pytest 6.2. `#2741 <https://github.com/pybind/pybind11/pull/2741>`_ Warning fixes: @@ -263,7 +1403,7 @@ Packaging / building improvements: `#2338 <https://github.com/pybind/pybind11/pull/2338>`_ and `#2370 <https://github.com/pybind/pybind11/pull/2370>`_ - * Full integration with CMake’s C++ standard system and compile features + * Full integration with CMake's C++ standard system and compile features replaces ``PYBIND11_CPP_STANDARD``. * Generated config file is now portable to different Python/compiler/CMake @@ -495,7 +1635,7 @@ v2.4.0 (Sep 19, 2019) `#1888 <https://github.com/pybind/pybind11/pull/1888>`_. * ``py::details::overload_cast_impl`` is available in C++11 mode, can be used - like ``overload_cast`` with an additional set of parantheses. + like ``overload_cast`` with an additional set of parentheses. `#1581 <https://github.com/pybind/pybind11/pull/1581>`_. * Fixed ``get_include()`` on Conda. @@ -817,6 +1957,7 @@ v2.2.0 (August 31, 2017) from cpp_module import CppBase1, CppBase2 + class PyDerived(CppBase1, CppBase2): def __init__(self): CppBase1.__init__(self) # C++ bases must be initialized explicitly @@ -1029,7 +2170,7 @@ v2.2.0 (August 31, 2017) * Intel C++ compiler compatibility fixes. `#937 <https://github.com/pybind/pybind11/pull/937>`_. -* Fixed implicit conversion of `py::enum_` to integer types on Python 2.7. +* Fixed implicit conversion of ``py::enum_`` to integer types on Python 2.7. `#821 <https://github.com/pybind/pybind11/pull/821>`_. * Added ``py::hash`` to fetch the hash value of Python objects, and diff --git a/docs/classes.rst b/docs/classes.rst index f3610ef3..4f2167da 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -44,20 +44,30 @@ interactive Python session demonstrating this example is shown below: % python >>> import example - >>> p = example.Pet('Molly') + >>> p = example.Pet("Molly") >>> print(p) <example.Pet object at 0x10cd98060> >>> p.getName() - u'Molly' - >>> p.setName('Charly') + 'Molly' + >>> p.setName("Charly") >>> p.getName() - u'Charly' + 'Charly' .. seealso:: Static member functions can be bound in the same way using :func:`class_::def_static`. +.. note:: + + Binding C++ types in unnamed namespaces (also known as anonymous namespaces) + works reliably on many platforms, but not all. The `XFAIL_CONDITION` in + tests/test_unnamed_namespace_a.py encodes the currently known conditions. + For background see `#4319 <https://github.com/pybind/pybind11/pull/4319>`_. + If portability is a concern, it is therefore not recommended to bind C++ + types in unnamed namespaces. It will be safest to manually pick unique + namespace names. + Keyword and default arguments ============================= It is possible to specify keyword and default arguments using the syntax @@ -122,12 +132,12 @@ This makes it possible to write .. code-block:: pycon - >>> p = example.Pet('Molly') + >>> p = example.Pet("Molly") >>> p.name - u'Molly' - >>> p.name = 'Charly' + 'Molly' + >>> p.name = "Charly" >>> p.name - u'Charly' + 'Charly' Now suppose that ``Pet::name`` was a private internal variable that can only be accessed via setters and getters. @@ -174,10 +184,10 @@ Native Python classes can pick up new attributes dynamically: .. code-block:: pycon >>> class Pet: - ... name = 'Molly' + ... name = "Molly" ... >>> p = Pet() - >>> p.name = 'Charly' # overwrite existing + >>> p.name = "Charly" # overwrite existing >>> p.age = 2 # dynamically add a new attribute By default, classes exported from C++ do not support this and the only writable @@ -195,7 +205,7 @@ Trying to set any other attribute results in an error: .. code-block:: pycon >>> p = example.Pet() - >>> p.name = 'Charly' # OK, attribute defined in C++ + >>> p.name = "Charly" # OK, attribute defined in C++ >>> p.age = 2 # fail AttributeError: 'Pet' object has no attribute 'age' @@ -213,7 +223,7 @@ Now everything works as expected: .. code-block:: pycon >>> p = example.Pet() - >>> p.name = 'Charly' # OK, overwrite value in C++ + >>> p.name = "Charly" # OK, overwrite value in C++ >>> p.age = 2 # OK, dynamically add a new attribute >>> p.__dict__ # just like a native Python class {'age': 2} @@ -280,11 +290,11 @@ expose fields and methods of both types: .. code-block:: pycon - >>> p = example.Dog('Molly') + >>> p = example.Dog("Molly") >>> p.name - u'Molly' + 'Molly' >>> p.bark() - u'woof!' + 'woof!' The C++ classes defined above are regular non-polymorphic types with an inheritance relationship. This is reflected in Python: @@ -332,7 +342,7 @@ will automatically recognize this: >>> type(p) PolymorphicDog # automatically downcast >>> p.bark() - u'woof!' + 'woof!' Given a pointer to a polymorphic base, pybind11 performs automatic downcasting to the actual derived type. Note that this goes beyond the usual situation in @@ -434,8 +444,7 @@ you can use ``py::detail::overload_cast_impl`` with an additional set of parenth .def("set", overload_cast_<int>()(&Pet::set), "Set the pet's age") .def("set", overload_cast_<const std::string &>()(&Pet::set), "Set the pet's name"); -.. [#cpp14] A compiler which supports the ``-std=c++14`` flag - or Visual Studio 2015 Update 2 and newer. +.. [#cpp14] A compiler which supports the ``-std=c++14`` flag. .. note:: @@ -446,8 +455,7 @@ you can use ``py::detail::overload_cast_impl`` with an additional set of parenth Enumerations and internal types =============================== -Let's now suppose that the example class contains an internal enumeration type, -e.g.: +Let's now suppose that the example class contains internal types like enumerations, e.g.: .. code-block:: cpp @@ -457,10 +465,15 @@ e.g.: Cat }; + struct Attributes { + float age = 0; + }; + Pet(const std::string &name, Kind type) : name(name), type(type) { } std::string name; Kind type; + Attributes attr; }; The binding code for this example looks as follows: @@ -471,22 +484,28 @@ The binding code for this example looks as follows: pet.def(py::init<const std::string &, Pet::Kind>()) .def_readwrite("name", &Pet::name) - .def_readwrite("type", &Pet::type); + .def_readwrite("type", &Pet::type) + .def_readwrite("attr", &Pet::attr); py::enum_<Pet::Kind>(pet, "Kind") .value("Dog", Pet::Kind::Dog) .value("Cat", Pet::Kind::Cat) .export_values(); -To ensure that the ``Kind`` type is created within the scope of ``Pet``, the -``pet`` :class:`class_` instance must be supplied to the :class:`enum_`. + py::class_<Pet::Attributes>(pet, "Attributes") + .def(py::init<>()) + .def_readwrite("age", &Pet::Attributes::age); + + +To ensure that the nested types ``Kind`` and ``Attributes`` are created within the scope of ``Pet``, the +``pet`` :class:`class_` instance must be supplied to the :class:`enum_` and :class:`class_` constructor. The :func:`enum_::export_values` function exports the enum entries into the parent scope, which should be skipped for newer C++11-style strongly typed enums. .. code-block:: pycon - >>> p = Pet('Lucy', Pet.Cat) + >>> p = Pet("Lucy", Pet.Cat) >>> p.type Kind.Cat >>> int(p.type) @@ -508,7 +527,7 @@ The ``name`` property returns the name of the enum value as a unicode string. .. code-block:: pycon - >>> p = Pet( "Lucy", Pet.Cat ) + >>> p = Pet("Lucy", Pet.Cat) >>> pet_type = p.type >>> pet_type Pet.Cat @@ -530,3 +549,7 @@ The ``name`` property returns the name of the enum value as a unicode string. ... By default, these are omitted to conserve space. + +.. warning:: + + Contrary to Python customs, enum values from the wrappers should not be compared using ``is``, but with ``==`` (see `#1177 <https://github.com/pybind/pybind11/issues/1177>`_ for background). diff --git a/docs/compiling.rst b/docs/compiling.rst index 3a8a270d..1fd098be 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -42,10 +42,7 @@ An example of a ``setup.py`` using pybind11's helpers: ), ] - setup( - ..., - ext_modules=ext_modules - ) + setup(..., ext_modules=ext_modules) If you want to do an automatic search for the highest supported C++ standard, that is supported via a ``build_ext`` command override; it will only affect @@ -64,11 +61,20 @@ that is supported via a ``build_ext`` command override; it will only affect ), ] - setup( - ..., - cmdclass={"build_ext": build_ext}, - ext_modules=ext_modules - ) + setup(..., cmdclass={"build_ext": build_ext}, ext_modules=ext_modules) + +If you have single-file extension modules that are directly stored in the +Python source tree (``foo.cpp`` in the same directory as where a ``foo.py`` +would be located), you can also generate ``Pybind11Extensions`` using +``setup_helpers.intree_extensions``: ``intree_extensions(["path/to/foo.cpp", +...])`` returns a list of ``Pybind11Extensions`` which can be passed to +``ext_modules``, possibly after further customizing their attributes +(``libraries``, ``include_dirs``, etc.). By doing so, a ``foo.*.so`` extension +module will be generated and made available upon installation. + +``intree_extension`` will automatically detect if you are using a ``src``-style +layout (as long as no namespace packages are involved), but you can also +explicitly pass ``package_dir`` to it (as in ``setuptools.setup``). Since pybind11 does not require NumPy when building, a light-weight replacement for NumPy's parallel compilation distutils tool is included. Use it like this: @@ -93,14 +99,14 @@ to a memory dependent number. If you are developing rapidly and have a lot of C++ files, you may want to avoid rebuilding files that have not changed. For simple cases were you are using ``pip install -e .`` and do not have local headers, you can skip the -rebuild if a object file is newer than it's source (headers are not checked!) +rebuild if an object file is newer than its source (headers are not checked!) with the following: .. code-block:: python from pybind11.setup_helpers import ParallelCompile, naive_recompile - SmartCompile("NPY_NUM_BUILD_JOBS", needs_recompile=naive_recompile).install() + ParallelCompile("NPY_NUM_BUILD_JOBS", needs_recompile=naive_recompile).install() If you have a more complex build, you can implement a smarter function and pass @@ -149,7 +155,7 @@ Your ``pyproject.toml`` file will likely look something like this: and ``pyproject.toml`` are not even contained in the wheel, so this high Pip requirement is only for source builds, and will not affect users of your binary wheels. If you are building SDists and wheels, then - `pypa-build`_ is the recommended offical tool. + `pypa-build`_ is the recommended official tool. .. _PEP 517: https://www.python.org/dev/peps/pep-0517/ .. _cibuildwheel: https://cibuildwheel.readthedocs.io @@ -235,7 +241,7 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) add_subdirectory(pybind11) @@ -255,6 +261,9 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. versionchanged:: 2.6 CMake 3.4+ is required. +.. versionchanged:: 2.11 + CMake 3.5+ is required. + Further information can be found at :doc:`cmake/index`. pybind11_add_module @@ -334,7 +343,7 @@ standard explicitly with set(CMAKE_CXX_STANDARD 14 CACHE STRING "C++ version selection") # or 11, 14, 17, 20 set(CMAKE_CXX_STANDARD_REQUIRED ON) # optional, ensure standard is supported - set(CMAKE_CXX_EXTENSIONS OFF) # optional, keep compiler extensionsn off + set(CMAKE_CXX_EXTENSIONS OFF) # optional, keep compiler extensions off The variables can also be set when calling CMake from the command line using the ``-D<variable>=<value>`` flag. You can also manually set ``CXX_STANDARD`` @@ -411,10 +420,10 @@ existing targets instead: .. code-block:: cmake - cmake_minumum_required(VERSION 3.15...3.19) + cmake_minimum_required(VERSION 3.15...3.22) project(example LANGUAGES CXX) - find_package(Python COMPONENTS Interpreter Development REQUIRED) + find_package(Python 3.6 COMPONENTS Interpreter Development REQUIRED) find_package(pybind11 CONFIG REQUIRED) # or add_subdirectory(pybind11) @@ -427,9 +436,8 @@ algorithms from the CMake invocation, with ``-DPYBIND11_FINDPYTHON=ON``. .. warning:: - If you use FindPython2 and FindPython3 to dual-target Python, use the - individual targets listed below, and avoid targets that directly include - Python parts. + If you use FindPython to multi-target Python versions, use the individual + targets listed below, and avoid targets that directly include Python parts. There are `many ways to hint or force a discovery of a specific Python installation <https://cmake.org/cmake/help/latest/module/FindPython.html>`_), @@ -456,11 +464,8 @@ available in all modes. The targets provided are: ``pybind11::headers`` Just the pybind11 headers and minimum compile requirements - ``pybind11::python2_no_register`` - Quiets the warning/error when mixing C++14 or higher and Python 2 - ``pybind11::pybind11`` - Python headers + ``pybind11::headers`` + ``pybind11::python2_no_register`` (Python 2 only) + Python headers + ``pybind11::headers`` ``pybind11::python_link_helper`` Just the "linking" part of pybind11:module @@ -469,7 +474,7 @@ available in all modes. The targets provided are: Everything for extension modules - ``pybind11::pybind11`` + ``Python::Module`` (FindPython CMake 3.15+) or ``pybind11::python_link_helper`` ``pybind11::embed`` - Everything for embedding the Python interpreter - ``pybind11::pybind11`` + ``Python::Embed`` (FindPython) or Python libs + Everything for embedding the Python interpreter - ``pybind11::pybind11`` + ``Python::Python`` (FindPython) or Python libs ``pybind11::lto`` / ``pybind11::thin_lto`` An alternative to `INTERPROCEDURAL_OPTIMIZATION` for adding link-time optimization. @@ -493,7 +498,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -503,7 +508,10 @@ You can use these targets to build complex applications. For example, the target_link_libraries(example PRIVATE pybind11::module pybind11::lto pybind11::windows_extras) pybind11_extension(example) - pybind11_strip(example) + if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Strip unnecessary sections of the binary on Linux/macOS + pybind11_strip(example) + endif() set_target_properties(example PROPERTIES CXX_VISIBILITY_PRESET "hidden" CUDA_VISIBILITY_PRESET "hidden") @@ -516,7 +524,7 @@ Instead of setting properties, you can set ``CMAKE_*`` variables to initialize t compiler flags are provided to ensure high quality code generation. In contrast to the ``pybind11_add_module()`` command, the CMake interface provides a *composable* set of targets to ensure that you retain flexibility. - It can be expecially important to provide or set these properties; the + It can be especially important to provide or set these properties; the :ref:`FAQ <faq:symhidden>` contains an explanation on why these are needed. .. versionadded:: 2.6 @@ -548,7 +556,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -571,21 +579,12 @@ On Linux, you can compile an example such as the one given in $ c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) example.cpp -o example$(python3-config --extension-suffix) -The flags given here assume that you're using Python 3. For Python 2, just -change the executable appropriately (to ``python`` or ``python2``). - The ``python3 -m pybind11 --includes`` command fetches the include paths for both pybind11 and Python headers. This assumes that pybind11 has been installed using ``pip`` or ``conda``. If it hasn't, you can also manually specify ``-I <path-to-pybind11>/include`` together with the Python includes path ``python3-config --includes``. -Note that Python 2.7 modules don't use a special suffix, so you should simply -use ``example.so`` instead of ``example$(python3-config --extension-suffix)``. -Besides, the ``--extension-suffix`` option may or may not be available, depending -on the distribution; in the latter case, the module extension can be manually -set to ``.so``. - On macOS: the build command is almost the same but it also requires passing the ``-undefined dynamic_lookup`` flag so as to ignore missing symbols when building the module: diff --git a/docs/conf.py b/docs/conf.py index 6ac054c6..6e24751e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # pybind11 documentation build configuration file, created by # sphinx-quickstart on Sun Oct 11 19:23:48 2015. @@ -13,12 +12,11 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os -import shlex +import re import subprocess +import sys from pathlib import Path -import re DIR = Path(__file__).parent.resolve() @@ -37,6 +35,7 @@ DIR = Path(__file__).parent.resolve() # ones. extensions = [ "breathe", + "sphinx_copybutton", "sphinxcontrib.rsvgconverter", "sphinxcontrib.moderncmakedomain", ] @@ -127,23 +126,7 @@ todo_include_todos = False # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -on_rtd = os.environ.get("READTHEDOCS", None) == "True" - -if not on_rtd: # only import and set the theme if we're building docs locally - import sphinx_rtd_theme - - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - - html_context = {"css_files": ["_static/theme_overrides.css"]} -else: - html_context = { - "css_files": [ - "//media.readthedocs.org/css/sphinx_rtd_theme.css", - "//media.readthedocs.org/css/readthedocs-doc-embed.css", - "_static/theme_overrides.css", - ] - } +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -174,6 +157,10 @@ else: # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +html_css_files = [ + "css/custom.css", +] + # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. @@ -346,9 +333,9 @@ def generate_doxygen_xml(app): subprocess.call(["doxygen", "--version"]) retcode = subprocess.call(["doxygen"], cwd=app.confdir) if retcode < 0: - sys.stderr.write("doxygen error code: {}\n".format(-retcode)) + sys.stderr.write(f"doxygen error code: {-retcode}\n") except OSError as e: - sys.stderr.write("doxygen execution failed: {}\n".format(e)) + sys.stderr.write(f"doxygen execution failed: {e}\n") def prepare(app): @@ -366,12 +353,11 @@ def prepare(app): f.write(contents) -def clean_up(app, exception): +def clean_up(app, exception): # noqa: ARG001 (DIR / "readme.rst").unlink() def setup(app): - # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) diff --git a/docs/faq.rst b/docs/faq.rst index 8bf05a40..1eb00efa 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -5,12 +5,10 @@ Frequently asked questions =========================================================== 1. Make sure that the name specified in PYBIND11_MODULE is identical to the -filename of the extension library (without suffixes such as .so) +filename of the extension library (without suffixes such as ``.so``). 2. If the above did not fix the issue, you are likely using an incompatible -version of Python (for instance, the extension library was compiled against -Python 2, while the interpreter is running on top of some version of Python -3, or vice versa). +version of Python that does not match what you compiled with. "Symbol not found: ``__Py_ZeroStruct`` / ``_PyInstanceMethod_Type``" ======================================================================== @@ -54,7 +52,7 @@ provided by the caller -- in fact, it does nothing at all. .. code-block:: python def increment(i): - i += 1 # nope.. + i += 1 # nope.. pybind11 is also affected by such language-level conventions, which means that binding ``increment`` or ``increment_ptr`` will also create Python functions @@ -147,7 +145,7 @@ using C++14 template metaprogramming. .. _`faq:hidden_visibility`: -"‘SomeClass’ declared with greater visibility than the type of its field ‘SomeClass::member’ [-Wattributes]" +"'SomeClass' declared with greater visibility than the type of its field 'SomeClass::member' [-Wattributes]" ============================================================================================================ This error typically indicates that you are compiling without the required @@ -169,8 +167,8 @@ can be changed, but even if it isn't it is not always enough to guarantee complete independence of the symbols involved when not using ``-fvisibility=hidden``. -Additionally, ``-fvisiblity=hidden`` can deliver considerably binary size -savings. (See the following section for more details). +Additionally, ``-fvisibility=hidden`` can deliver considerably binary size +savings. (See the following section for more details.) .. _`faq:symhidden`: @@ -180,7 +178,7 @@ How can I create smaller binaries? To do its job, pybind11 extensively relies on a programming technique known as *template metaprogramming*, which is a way of performing computation at compile -time using type information. Template metaprogamming usually instantiates code +time using type information. Template metaprogramming usually instantiates code involving significant numbers of deeply nested types that are either completely removed or reduced to just a few instructions during the compiler's optimization phase. However, due to the nested nature of these types, the resulting symbol @@ -222,20 +220,6 @@ In addition to decreasing binary size, ``-fvisibility=hidden`` also avoids potential serious issues when loading multiple modules and is required for proper pybind operation. See the previous FAQ entry for more details. -Working with ancient Visual Studio 2008 builds on Windows -========================================================= - -The official Windows distributions of Python are compiled using truly -ancient versions of Visual Studio that lack good C++11 support. Some users -implicitly assume that it would be impossible to load a plugin built with -Visual Studio 2015 into a Python distribution that was compiled using Visual -Studio 2008. However, no such issue exists: it's perfectly legitimate to -interface DLLs that are built with different compilers and/or C libraries. -Common gotchas to watch out for involve not ``free()``-ing memory region -that that were ``malloc()``-ed in another shared library, using data -structures with incompatible ABIs, and so on. pybind11 is very careful not -to make these types of mistakes. - How can I properly handle Ctrl-C in long-running functions? =========================================================== @@ -289,27 +273,7 @@ Conflicts can arise, however, when using pybind11 in a project that *also* uses the CMake Python detection in a system with several Python versions installed. This difference may cause inconsistencies and errors if *both* mechanisms are -used in the same project. Consider the following CMake code executed in a -system with Python 2.7 and 3.x installed: - -.. code-block:: cmake - - find_package(PythonInterp) - find_package(PythonLibs) - find_package(pybind11) - -It will detect Python 2.7 and pybind11 will pick it as well. - -In contrast this code: - -.. code-block:: cmake - - find_package(pybind11) - find_package(PythonInterp) - find_package(PythonLibs) - -will detect Python 3.x for pybind11 and may crash on -``find_package(PythonLibs)`` afterwards. +used in the same project. There are three possible solutions: @@ -320,7 +284,8 @@ There are three possible solutions: COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead of the old, deprecated search tools, and these modules are much better at - finding the correct Python. + finding the correct Python. If FindPythonLibs/Interp are not available + (CMake 3.27+), then this will be ignored and FindPython will be used. 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. However, you will have to use the target-based system, and do more setup yourself, because it does not know about or include things that depend on diff --git a/docs/limitations.rst b/docs/limitations.rst index be7300cd..def5ad65 100644 --- a/docs/limitations.rst +++ b/docs/limitations.rst @@ -57,16 +57,16 @@ clean, well written patch would likely be accepted to solve them. Python 3.9.0 warning ^^^^^^^^^^^^^^^^^^^^ -Combining older versions of pybind11 (< 2.6.0) with Python on 3.9.0 will -trigger undefined behavior that typically manifests as crashes during +Combining older versions of pybind11 (< 2.6.0) with Python on exactly 3.9.0 +will trigger undefined behavior that typically manifests as crashes during interpreter shutdown (but could also destroy your data. **You have been warned**). -This issue has been -`fixed in Python <https://github.com/python/cpython/pull/22670>`_. As a -mitigation until 3.9.1 is released and commonly used, pybind11 (2.6.0 or newer) -includes a temporary workaround specifically when Python 3.9.0 is detected at -runtime, leaking about 50 bytes of memory when a callback function is garbage -collected. For reference; the pybind11 test suite has about 2,000 such -callbacks, but only 49 are garbage collected before the end-of-process. Wheels -built with Python 3.9.0 will correctly avoid the leak when run in Python 3.9.1. +This issue was `fixed in Python <https://github.com/python/cpython/pull/22670>`_. +As a mitigation for this bug, pybind11 2.6.0 or newer includes a workaround +specifically when Python 3.9.0 is detected at runtime, leaking about 50 bytes +of memory when a callback function is garbage collected. For reference, the +pybind11 test suite has about 2,000 such callbacks, but only 49 are garbage +collected before the end-of-process. Wheels (even if built with Python 3.9.0) +will correctly avoid the leak when run in Python 3.9.1, and this does not +affect other 3.X versions. diff --git a/docs/pybind11-logo.png b/docs/pybind11-logo.png Binary files differindex 4cbad54f..2d633a4d 100644 --- a/docs/pybind11-logo.png +++ b/docs/pybind11-logo.png diff --git a/docs/reference.rst b/docs/reference.rst index e3a61afb..e64a0351 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -52,6 +52,20 @@ Convenience classes for specific Python types .. doxygengroup:: pytypes :members: +Convenience functions converting to Python types +================================================ + +.. doxygenfunction:: make_tuple(Args&&...) + +.. doxygenfunction:: make_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_iterator(Type &, Extra&&...) + +.. doxygenfunction:: make_key_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_key_iterator(Type &, Extra&&...) + +.. doxygenfunction:: make_value_iterator(Iterator, Sentinel, Extra &&...) +.. doxygenfunction:: make_value_iterator(Type &, Extra&&...) + .. _extras: Passing extra arguments to ``def`` or ``class_`` @@ -110,7 +124,6 @@ Exceptions .. doxygenclass:: builtin_exception :members: - Literals ======== diff --git a/docs/release.rst b/docs/release.rst index 9dbff038..4950c3b8 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -15,31 +15,39 @@ For example: For beta, ``PYBIND11_VERSION_PATCH`` should be ``Z.b1``. RC's can be ``Z.rc1``. Always include the dot (even though PEP 440 allows it to be dropped). For a -final release, this must be a simple integer. +final release, this must be a simple integer. There is also a HEX version of +the version just below. To release a new version of pybind11: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +If you don't have nox, you should either use ``pipx run nox`` instead, or use +``pipx install nox`` or ``brew install nox`` (Unix). + - Update the version number - - Update ``PYBIND11_VERSION_MAJOR`` etc. in - ``include/pybind11/detail/common.h``. PATCH should be a simple integer. - - Update ``pybind11/_version.py`` (match above) - - Ensure that all the information in ``setup.cfg`` is up-to-date, like - supported Python versions. - - Add release date in ``docs/changelog.rst``. - - Check to make sure - `needs-changelog <https://github.com/pybind/pybind11/pulls?q=is%3Apr+is%3Aclosed+label%3A%22needs+changelog%22>`_ - issues are entered in the changelog (clear the label when done). - - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it - fails due to a known flake issue, either ignore or restart CI.) + - Update ``PYBIND11_VERSION_MAJOR`` etc. in + ``include/pybind11/detail/common.h``. PATCH should be a simple integer. + - Update the version HEX just below, as well. + - Update ``pybind11/_version.py`` (match above) + - Run ``nox -s tests_packaging`` to ensure this was done correctly. + - Ensure that all the information in ``setup.cfg`` is up-to-date, like + supported Python versions. + - Add release date in ``docs/changelog.rst`` and integrate the output of + ``nox -s make_changelog``. + - Note that the ``make_changelog`` command inspects + `needs changelog <https://github.com/pybind/pybind11/pulls?q=is%3Apr+is%3Aclosed+label%3A%22needs+changelog%22>`_. + - Manually clear the ``needs changelog`` labels using the GitHub web + interface (very easy: start by clicking the link above). + - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it + fails due to a known flake issue, either ignore or restart CI.) - Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version - - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` - - Update branch: ``git checkout vX.Y``, ``git merge <release branch>``, ``git push`` + - New branch: ``git checkout -b vX.Y``, ``git push -u origin vX.Y`` + - Update branch: ``git checkout vX.Y``, ``git merge <release branch>``, ``git push`` - Update tags (optional; if you skip this, the GitHub release makes a - non-annotated tag for you) - - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``. - - ``git push --tags``. + non-annotated tag for you) + - ``git tag -a vX.Y.Z -m 'vX.Y.Z release'``. + - ``git push --tags``. - Update stable - ``git checkout stable`` - ``git merge master`` @@ -47,29 +55,30 @@ To release a new version of pybind11: - Make a GitHub release (this shows up in the UI, sends new release notifications to users watching releases, and also uploads PyPI packages). (Note: if you do not use an existing tag, this creates a new lightweight tag - for you, so you could skip the above step). - - GUI method: click "Create a new release" on the far right, fill in the tag - name (if you didn't tag above, it will be made here), fill in a release - name like "Version X.Y.Z", and optionally copy-and-paste the changelog into - the description (processed as markdown by Pandoc). Check "pre-release" if - this is a beta/RC. You can get partway there with - ``cat docs/changelog.rst | pandsoc -f rst -t markdown``. - - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` - If this is a pre-release, add ``-p``. + for you, so you could skip the above step.) + - GUI method: Under `releases <https://github.com/pybind/pybind11/releases>`_ + click "Draft a new release" on the far right, fill in the tag name + (if you didn't tag above, it will be made here), fill in a release name + like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog + into the description (usually ``cat docs/changelog.rst | pandoc -f rst -t gfm``). + Check "pre-release" if this is a beta/RC. + - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` + If this is a pre-release, add ``-p``. - Get back to work - - Make sure you are on master, not somewhere else: ``git checkout master`` - - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to - ``0.dev1`` and increment MINOR). - - Update ``_version.py`` to match - - Add a spot for in-development updates in ``docs/changelog.rst``. - - ``git add``, ``git commit``, ``git push`` + - Make sure you are on master, not somewhere else: ``git checkout master`` + - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to + ``0.dev1`` and increment MINOR). + - Update ``_version.py`` to match + - Run ``nox -s tests_packaging`` to ensure this was done correctly. + - Add a spot for in-development updates in ``docs/changelog.rst``. + - ``git add``, ``git commit``, ``git push`` If a version branch is updated, remember to set PATCH to ``1.dev1``. If you'd like to bump homebrew, run: -.. code-block:: +.. code-block:: console brew bump-formula-pr --url https://github.com/pybind/pybind11/archive/vX.Y.Z.tar.gz @@ -84,9 +93,7 @@ If you need to manually upload releases, you can download the releases from the .. code-block:: bash - python3 -m pip install build - python3 -m build - PYBIND11_SDIST_GLOBAL=1 python3 -m build + nox -s build twine upload dist/* This makes SDists and wheels, and the final line uploads them. diff --git a/docs/requirements.txt b/docs/requirements.txt index 9e6a3f63..d2a9ae16 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,6 @@ -breathe==4.25.1 -commonmark==0.9.1 -recommonmark==0.7.1 -sphinx==3.3.1 -sphinx_rtd_theme==0.5.0 -sphinxcontrib-moderncmakedomain==3.17 -sphinxcontrib-svg2pdfconverter==1.1.0 +breathe==4.34.0 +furo==2022.6.21 +sphinx==5.0.2 +sphinx-copybutton==0.5.0 +sphinxcontrib-moderncmakedomain==3.21.4 +sphinxcontrib-svg2pdfconverter==1.2.0 diff --git a/docs/upgrade.rst b/docs/upgrade.rst index f13fbcf5..b13d21f5 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,60 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.11: + +v2.11 +===== + +* The minimum version of CMake is now 3.5. A future version will likely move to + requiring something like CMake 3.15. Note that CMake 3.27 is removing the + long-deprecated support for ``FindPythonInterp`` if you set 3.27 as the + minimum or maximum supported version. To prepare for that future, CMake 3.15+ + using ``FindPython`` or setting ``PYBIND11_FINDPYTHON`` is highly recommended, + otherwise pybind11 will automatically switch to using ``FindPython`` if + ``FindPythonInterp`` is not available. + + +.. _upgrade-guide-2.9: + +v2.9 +==== + +* Any usage of the recently added ``py::make_simple_namespace`` should be + converted to using ``py::module_::import("types").attr("SimpleNamespace")`` + instead. + +* The use of ``_`` in custom type casters can now be replaced with the more + readable ``const_name`` instead. The old ``_`` shortcut has been retained + unless it is being used as a macro (like for gettext). + + +.. _upgrade-guide-2.7: + +v2.7 +==== + +*Before* v2.7, ``py::str`` can hold ``PyUnicodeObject`` or ``PyBytesObject``, +and ``py::isinstance<str>()`` is ``true`` for both ``py::str`` and +``py::bytes``. Starting with v2.7, ``py::str`` exclusively holds +``PyUnicodeObject`` (`#2409 <https://github.com/pybind/pybind11/pull/2409>`_), +and ``py::isinstance<str>()`` is ``true`` only for ``py::str``. To help in +the transition of user code, the ``PYBIND11_STR_LEGACY_PERMISSIVE`` macro +is provided as an escape hatch to go back to the legacy behavior. This macro +will be removed in future releases. Two types of required fixes are expected +to be common: + +* Accidental use of ``py::str`` instead of ``py::bytes``, masked by the legacy + behavior. These are probably very easy to fix, by changing from + ``py::str`` to ``py::bytes``. + +* Reliance on py::isinstance<str>(obj) being ``true`` for + ``py::bytes``. This is likely to be easy to fix in most cases by adding + ``|| py::isinstance<bytes>(obj)``, but a fix may be more involved, e.g. if + ``py::isinstance<T>`` appears in a template. Such situations will require + careful review and custom fixes. + + .. _upgrade-guide-2.6: v2.6 @@ -256,7 +310,7 @@ Within pybind11's CMake build system, ``pybind11_add_module`` has always been setting the ``-fvisibility=hidden`` flag in release mode. From now on, it's being applied unconditionally, even in debug mode and it can no longer be opted out of with the ``NO_EXTRAS`` option. The ``pybind11::module`` target now also -adds this flag to it's interface. The ``pybind11::embed`` target is unchanged. +adds this flag to its interface. The ``pybind11::embed`` target is unchanged. The most significant change here is for the ``pybind11::module`` target. If you were previously relying on default visibility, i.e. if your Python module was @@ -484,7 +538,7 @@ include a declaration of the form: PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>) -Continuing to do so won’t cause an error or even a deprecation warning, +Continuing to do so won't cause an error or even a deprecation warning, but it's completely redundant. diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 50efdc7c..1044db94 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -10,72 +10,116 @@ #pragma once +#include "detail/common.h" #include "cast.h" +#include <functional> + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) /// \addtogroup annotations /// @{ /// Annotation for methods -struct is_method { handle class_; is_method(const handle &c) : class_(c) { } }; +struct is_method { + handle class_; + explicit is_method(const handle &c) : class_(c) {} +}; + +/// Annotation for setters +struct is_setter {}; /// Annotation for operators -struct is_operator { }; +struct is_operator {}; /// Annotation for classes that cannot be subclassed -struct is_final { }; +struct is_final {}; /// Annotation for parent scope -struct scope { handle value; scope(const handle &s) : value(s) { } }; +struct scope { + handle value; + explicit scope(const handle &s) : value(s) {} +}; /// Annotation for documentation -struct doc { const char *value; doc(const char *value) : value(value) { } }; +struct doc { + const char *value; + explicit doc(const char *value) : value(value) {} +}; /// Annotation for function names -struct name { const char *value; name(const char *value) : value(value) { } }; +struct name { + const char *value; + explicit name(const char *value) : value(value) {} +}; /// Annotation indicating that a function is an overload associated with a given "sibling" -struct sibling { handle value; sibling(const handle &value) : value(value.ptr()) { } }; +struct sibling { + handle value; + explicit sibling(const handle &value) : value(value.ptr()) {} +}; /// Annotation indicating that a class derives from another given type -template <typename T> struct base { +template <typename T> +struct base { - PYBIND11_DEPRECATED("base<T>() was deprecated in favor of specifying 'T' as a template argument to class_") - base() { } // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute + PYBIND11_DEPRECATED( + "base<T>() was deprecated in favor of specifying 'T' as a template argument to class_") + base() = default; }; /// Keep patient alive while nurse lives -template <size_t Nurse, size_t Patient> struct keep_alive { }; +template <size_t Nurse, size_t Patient> +struct keep_alive {}; /// Annotation indicating that a class is involved in a multiple inheritance relationship -struct multiple_inheritance { }; +struct multiple_inheritance {}; /// Annotation which enables dynamic attributes, i.e. adds `__dict__` to a class -struct dynamic_attr { }; +struct dynamic_attr {}; /// Annotation which enables the buffer protocol for a type -struct buffer_protocol { }; +struct buffer_protocol {}; /// Annotation which requests that a special metaclass is created for a type struct metaclass { handle value; PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.") - metaclass() { } // NOLINT(modernize-use-equals-default): breaks MSVC 2015 when adding an attribute + metaclass() = default; /// Override pybind11's default metaclass - explicit metaclass(handle value) : value(value) { } + explicit metaclass(handle value) : value(value) {} +}; + +/// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that +/// may be used to customize the Python type. +/// +/// The callback is invoked immediately before `PyType_Ready`. +/// +/// Note: This is an advanced interface, and uses of it may require changes to +/// work with later versions of pybind11. You may wish to consult the +/// implementation of `make_new_python_type` in `detail/classes.h` to understand +/// the context in which the callback will be run. +struct custom_type_setup { + using callback = std::function<void(PyHeapTypeObject *heap_type)>; + + explicit custom_type_setup(callback value) : value(std::move(value)) {} + + callback value; }; /// Annotation that marks a class as local to the module: -struct module_local { const bool value; constexpr module_local(bool v = true) : value(v) { } }; +struct module_local { + const bool value; + constexpr explicit module_local(bool v = true) : value(v) {} +}; /// Annotation to mark enums as an arithmetic type -struct arithmetic { }; +struct arithmetic {}; /// Mark a function for addition at the beginning of the existing overload chain instead of the end -struct prepend { }; +struct prepend {}; /** \rst A call policy which places one or more guard variables (``Ts...``) around the function call. @@ -95,9 +139,13 @@ struct prepend { }; return foo(args...); // forwarded arguments }); \endrst */ -template <typename... Ts> struct call_guard; +template <typename... Ts> +struct call_guard; -template <> struct call_guard<> { using type = detail::void_type; }; +template <> +struct call_guard<> { + using type = detail::void_type; +}; template <typename T> struct call_guard<T> { @@ -122,8 +170,9 @@ PYBIND11_NAMESPACE_BEGIN(detail) enum op_id : int; enum op_type : int; struct undefined_t; -template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> struct op_; -inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); +template <op_id id, op_type ot, typename L = undefined_t, typename R = undefined_t> +struct op_; +void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); /// Internal data structure which holds metadata about a keyword argument struct argument_record { @@ -134,15 +183,16 @@ struct argument_record { bool none : 1; ///< True if None is allowed when loading argument_record(const char *name, const char *descr, handle value, bool convert, bool none) - : name(name), descr(descr), value(value), convert(convert), none(none) { } + : name(name), descr(descr), value(value), convert(convert), none(none) {} }; -/// Internal data structure which holds metadata about a bound function (signature, overloads, etc.) +/// Internal data structure which holds metadata about a bound function (signature, overloads, +/// etc.) struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), - has_kwargs(false), has_kw_only_args(false), prepend(false) { } + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -157,13 +207,13 @@ struct function_record { std::vector<argument_record> args; /// Pointer to lambda function which converts arguments and performs the actual call - handle (*impl) (function_call &) = nullptr; + handle (*impl)(function_call &) = nullptr; /// Storage for the wrapped function pointer and captured data, if any - void *data[3] = { }; + void *data[3] = {}; /// Pointer to custom destructor for 'data' (if needed) - void (*free_data) (function_record *ptr) = nullptr; + void (*free_data)(function_record *ptr) = nullptr; /// Return value policy associated with this function return_value_policy policy = return_value_policy::automatic; @@ -183,23 +233,24 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; /// True if the function has a '**kwargs' argument bool has_kwargs : 1; - /// True once a 'py::kw_only' is encountered (any following args are keyword-only) - bool has_kw_only_args : 1; - /// True if this function is to be inserted at the beginning of the overload resolution chain bool prepend : 1; /// Number of arguments (including py::args and/or py::kwargs, if present) std::uint16_t nargs; - /// Number of trailing arguments (counted in `nargs`) that are keyword-only - std::uint16_t nargs_kw_only = 0; + /// Number of leading positional arguments, which are terminated by a py::args or py::kwargs + /// argument or by a py::kw_only annotation. + std::uint16_t nargs_pos = 0; /// Number of leading arguments (counted in `nargs`) that are positional-only std::uint16_t nargs_pos_only = 0; @@ -221,7 +272,7 @@ struct function_record { struct type_record { PYBIND11_NOINLINE type_record() : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), - default_holder(true), module_local(false), is_final(false) { } + default_holder(true), module_local(false), is_final(false) {} /// Handle to the parent scope handle scope; @@ -259,6 +310,9 @@ struct type_record { /// Custom metaclass (optional) handle metaclass; + /// Custom type setup. + custom_type_setup::callback custom_type_setup_callback; + /// Multiple inheritance marker bool multiple_inheritance : 1; @@ -277,42 +331,45 @@ struct type_record { /// Is the class inheritable from python classes? bool is_final : 1; - PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *)) { - auto base_info = detail::get_type_info(base, false); + PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *) ) { + auto *base_info = detail::get_type_info(base, false); if (!base_info) { std::string tname(base.name()); detail::clean_type_id(tname); - pybind11_fail("generic_type: type \"" + std::string(name) + - "\" referenced unknown base type \"" + tname + "\""); + pybind11_fail("generic_type: type \"" + std::string(name) + + "\" referenced unknown base type \"" + tname + "\""); } if (default_holder != base_info->default_holder) { std::string tname(base.name()); detail::clean_type_id(tname); - pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + - (default_holder ? "does not have" : "has") + - " a non-default holder type while its base \"" + tname + "\" " + - (base_info->default_holder ? "does not" : "does")); + pybind11_fail("generic_type: type \"" + std::string(name) + "\" " + + (default_holder ? "does not have" : "has") + + " a non-default holder type while its base \"" + tname + "\" " + + (base_info->default_holder ? "does not" : "does")); } bases.append((PyObject *) base_info->type); - if (base_info->type->tp_dictoffset != 0) - dynamic_attr = true; +#if PY_VERSION_HEX < 0x030B0000 + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#else + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif - if (caster) + if (caster) { base_info->implicit_casts.emplace_back(type, caster); + } } }; -inline function_call::function_call(const function_record &f, handle p) : - func(f), parent(p) { +inline function_call::function_call(const function_record &f, handle p) : func(f), parent(p) { args.reserve(f.nargs); args_convert.reserve(f.nargs); } /// Tag for a new-style `__init__` defined in `detail/init.h` -struct is_new_style_constructor { }; +struct is_new_style_constructor {}; /** * Partial template specializations to process custom attributes provided to @@ -320,129 +377,183 @@ struct is_new_style_constructor { }; * fields in the type_record and function_record data structures or executed at * runtime to deal with custom call policies (e.g. keep_alive). */ -template <typename T, typename SFINAE = void> struct process_attribute; +template <typename T, typename SFINAE = void> +struct process_attribute; -template <typename T> struct process_attribute_default { +template <typename T> +struct process_attribute_default { /// Default implementation: do nothing - static void init(const T &, function_record *) { } - static void init(const T &, type_record *) { } - static void precall(function_call &) { } - static void postcall(function_call &, handle) { } + static void init(const T &, function_record *) {} + static void init(const T &, type_record *) {} + static void precall(function_call &) {} + static void postcall(function_call &, handle) {} }; /// Process an attribute specifying the function's name -template <> struct process_attribute<name> : process_attribute_default<name> { +template <> +struct process_attribute<name> : process_attribute_default<name> { static void init(const name &n, function_record *r) { r->name = const_cast<char *>(n.value); } }; /// Process an attribute specifying the function's docstring -template <> struct process_attribute<doc> : process_attribute_default<doc> { +template <> +struct process_attribute<doc> : process_attribute_default<doc> { static void init(const doc &n, function_record *r) { r->doc = const_cast<char *>(n.value); } }; /// Process an attribute specifying the function's docstring (provided as a C-style string) -template <> struct process_attribute<const char *> : process_attribute_default<const char *> { +template <> +struct process_attribute<const char *> : process_attribute_default<const char *> { static void init(const char *d, function_record *r) { r->doc = const_cast<char *>(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast<char *>(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; -template <> struct process_attribute<char *> : process_attribute<const char *> { }; +template <> +struct process_attribute<char *> : process_attribute<const char *> {}; /// Process an attribute indicating the function's return value policy -template <> struct process_attribute<return_value_policy> : process_attribute_default<return_value_policy> { +template <> +struct process_attribute<return_value_policy> : process_attribute_default<return_value_policy> { static void init(const return_value_policy &p, function_record *r) { r->policy = p; } }; -/// Process an attribute which indicates that this is an overloaded function associated with a given sibling -template <> struct process_attribute<sibling> : process_attribute_default<sibling> { +/// Process an attribute which indicates that this is an overloaded function associated with a +/// given sibling +template <> +struct process_attribute<sibling> : process_attribute_default<sibling> { static void init(const sibling &s, function_record *r) { r->sibling = s.value; } }; /// Process an attribute which indicates that this function is a method -template <> struct process_attribute<is_method> : process_attribute_default<is_method> { - static void init(const is_method &s, function_record *r) { r->is_method = true; r->scope = s.class_; } +template <> +struct process_attribute<is_method> : process_attribute_default<is_method> { + static void init(const is_method &s, function_record *r) { + r->is_method = true; + r->scope = s.class_; + } +}; + +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute<is_setter> : process_attribute_default<is_setter> { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } }; /// Process an attribute which indicates the parent scope of a method -template <> struct process_attribute<scope> : process_attribute_default<scope> { +template <> +struct process_attribute<scope> : process_attribute_default<scope> { static void init(const scope &s, function_record *r) { r->scope = s.value; } }; /// Process an attribute which indicates that this function is an operator -template <> struct process_attribute<is_operator> : process_attribute_default<is_operator> { +template <> +struct process_attribute<is_operator> : process_attribute_default<is_operator> { static void init(const is_operator &, function_record *r) { r->is_operator = true; } }; -template <> struct process_attribute<is_new_style_constructor> : process_attribute_default<is_new_style_constructor> { - static void init(const is_new_style_constructor &, function_record *r) { r->is_new_style_constructor = true; } +template <> +struct process_attribute<is_new_style_constructor> + : process_attribute_default<is_new_style_constructor> { + static void init(const is_new_style_constructor &, function_record *r) { + r->is_new_style_constructor = true; + } }; -inline void process_kw_only_arg(const arg &a, function_record *r) { - if (!a.name || strlen(a.name) == 0) - pybind11_fail("arg(): cannot specify an unnamed argument after an kw_only() annotation"); - ++r->nargs_kw_only; +inline void check_kw_only_arg(const arg &a, function_record *r) { + if (r->args.size() > r->nargs_pos && (!a.name || a.name[0] == '\0')) { + pybind11_fail("arg(): cannot specify an unnamed argument after a kw_only() annotation or " + "args() argument"); + } +} + +inline void append_self_arg_if_needed(function_record *r) { + if (r->is_method && r->args.empty()) { + r->args.emplace_back("self", nullptr, handle(), /*convert=*/true, /*none=*/false); + } } /// Process a keyword argument attribute (*without* a default value) -template <> struct process_attribute<arg> : process_attribute_default<arg> { +template <> +struct process_attribute<arg> : process_attribute_default<arg> { static void init(const arg &a, function_record *r) { - if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr, handle(), true /*convert*/, false /*none not allowed*/); + append_self_arg_if_needed(r); r->args.emplace_back(a.name, nullptr, handle(), !a.flag_noconvert, a.flag_none); - if (r->has_kw_only_args) process_kw_only_arg(a, r); + check_kw_only_arg(a, r); } }; /// Process a keyword argument attribute (*with* a default value) -template <> struct process_attribute<arg_v> : process_attribute_default<arg_v> { +template <> +struct process_attribute<arg_v> : process_attribute_default<arg_v> { static void init(const arg_v &a, function_record *r) { - if (r->is_method && r->args.empty()) - r->args.emplace_back("self", nullptr /*descr*/, handle() /*parent*/, true /*convert*/, false /*none not allowed*/); + if (r->is_method && r->args.empty()) { + r->args.emplace_back( + "self", /*descr=*/nullptr, /*parent=*/handle(), /*convert=*/true, /*none=*/false); + } if (!a.value) { -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) std::string descr("'"); - if (a.name) descr += std::string(a.name) + ": "; + if (a.name) { + descr += std::string(a.name) + ": "; + } descr += a.type + "'"; if (r->is_method) { - if (r->name) - descr += " in method '" + (std::string) str(r->scope) + "." + (std::string) r->name + "'"; - else + if (r->name) { + descr += " in method '" + (std::string) str(r->scope) + "." + + (std::string) r->name + "'"; + } else { descr += " in method of '" + (std::string) str(r->scope) + "'"; + } } else if (r->name) { descr += " in function '" + (std::string) r->name + "'"; } - pybind11_fail("arg(): could not convert default argument " - + descr + " into a Python object (type not registered yet?)"); + pybind11_fail("arg(): could not convert default argument " + descr + + " into a Python object (type not registered yet?)"); #else pybind11_fail("arg(): could not convert default argument " "into a Python object (type not registered yet?). " - "Compile in debug mode for more information."); + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "more information."); #endif } r->args.emplace_back(a.name, a.descr, a.value.inc_ref(), !a.flag_noconvert, a.flag_none); - if (r->has_kw_only_args) process_kw_only_arg(a, r); + check_kw_only_arg(a, r); } }; /// Process a keyword-only-arguments-follow pseudo argument -template <> struct process_attribute<kw_only> : process_attribute_default<kw_only> { +template <> +struct process_attribute<kw_only> : process_attribute_default<kw_only> { static void init(const kw_only &, function_record *r) { - r->has_kw_only_args = true; + append_self_arg_if_needed(r); + if (r->has_args && r->nargs_pos != static_cast<std::uint16_t>(r->args.size())) { + pybind11_fail("Mismatched args() and kw_only(): they must occur at the same relative " + "argument location (or omit kw_only() entirely)"); + } + r->nargs_pos = static_cast<std::uint16_t>(r->args.size()); } }; /// Process a positional-only-argument maker -template <> struct process_attribute<pos_only> : process_attribute_default<pos_only> { +template <> +struct process_attribute<pos_only> : process_attribute_default<pos_only> { static void init(const pos_only &, function_record *r) { + append_self_arg_if_needed(r); r->nargs_pos_only = static_cast<std::uint16_t>(r->args.size()); + if (r->nargs_pos_only > r->nargs_pos) { + pybind11_fail("pos_only(): cannot follow a py::args() argument"); + } + // It also can't follow a kw_only, but a static_assert in pybind11.h checks that } }; -/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees that) +/// Process a parent class attribute. Single inheritance only (class_ itself already guarantees +/// that) template <typename T> -struct process_attribute<T, enable_if_t<is_pyobject<T>::value>> : process_attribute_default<handle> { +struct process_attribute<T, enable_if_t<is_pyobject<T>::value>> + : process_attribute_default<handle> { static void init(const handle &h, type_record *r) { r->bases.append(h); } }; @@ -455,7 +566,9 @@ struct process_attribute<base<T>> : process_attribute_default<base<T>> { /// Process a multiple inheritance attribute template <> struct process_attribute<multiple_inheritance> : process_attribute_default<multiple_inheritance> { - static void init(const multiple_inheritance &, type_record *r) { r->multiple_inheritance = true; } + static void init(const multiple_inheritance &, type_record *r) { + r->multiple_inheritance = true; + } }; template <> @@ -464,6 +577,13 @@ struct process_attribute<dynamic_attr> : process_attribute_default<dynamic_attr> }; template <> +struct process_attribute<custom_type_setup> { + static void init(const custom_type_setup &value, type_record *r) { + r->custom_type_setup_callback = value.value; + } +}; + +template <> struct process_attribute<is_final> : process_attribute_default<is_final> { static void init(const is_final &, type_record *r) { r->is_final = true; } }; @@ -494,41 +614,59 @@ template <> struct process_attribute<arithmetic> : process_attribute_default<arithmetic> {}; template <typename... Ts> -struct process_attribute<call_guard<Ts...>> : process_attribute_default<call_guard<Ts...>> { }; +struct process_attribute<call_guard<Ts...>> : process_attribute_default<call_guard<Ts...>> {}; /** * Process a keep_alive call policy -- invokes keep_alive_impl during the * pre-call handler if both Nurse, Patient != 0 and use the post-call handler * otherwise */ -template <size_t Nurse, size_t Patient> struct process_attribute<keep_alive<Nurse, Patient>> : public process_attribute_default<keep_alive<Nurse, Patient>> { +template <size_t Nurse, size_t Patient> +struct process_attribute<keep_alive<Nurse, Patient>> + : public process_attribute_default<keep_alive<Nurse, Patient>> { template <size_t N = Nurse, size_t P = Patient, enable_if_t<N != 0 && P != 0, int> = 0> - static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); } + static void precall(function_call &call) { + keep_alive_impl(Nurse, Patient, call, handle()); + } template <size_t N = Nurse, size_t P = Patient, enable_if_t<N != 0 && P != 0, int> = 0> - static void postcall(function_call &, handle) { } + static void postcall(function_call &, handle) {} template <size_t N = Nurse, size_t P = Patient, enable_if_t<N == 0 || P == 0, int> = 0> - static void precall(function_call &) { } + static void precall(function_call &) {} template <size_t N = Nurse, size_t P = Patient, enable_if_t<N == 0 || P == 0, int> = 0> - static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); } + static void postcall(function_call &call, handle ret) { + keep_alive_impl(Nurse, Patient, call, ret); + } }; /// Recursively iterate over variadic template arguments -template <typename... Args> struct process_attributes { - static void init(const Args&... args, function_record *r) { - int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::init(args, r), 0) ... }; - ignore_unused(unused); +template <typename... Args> +struct process_attributes { + static void init(const Args &...args, function_record *r) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); + using expander = int[]; + (void) expander{ + 0, ((void) process_attribute<typename std::decay<Args>::type>::init(args, r), 0)...}; } - static void init(const Args&... args, type_record *r) { - int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::init(args, r), 0) ... }; - ignore_unused(unused); + static void init(const Args &...args, type_record *r) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(r); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(r); + using expander = int[]; + (void) expander{0, + (process_attribute<typename std::decay<Args>::type>::init(args, r), 0)...}; } static void precall(function_call &call) { - int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::precall(call), 0) ... }; - ignore_unused(unused); + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(call); + using expander = int[]; + (void) expander{0, + (process_attribute<typename std::decay<Args>::type>::precall(call), 0)...}; } static void postcall(function_call &call, handle fn_ret) { - int unused[] = { 0, (process_attribute<typename std::decay<Args>::type>::postcall(call, fn_ret), 0) ... }; - ignore_unused(unused); + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(call, fn_ret); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(fn_ret); + using expander = int[]; + (void) expander{ + 0, (process_attribute<typename std::decay<Args>::type>::postcall(call, fn_ret), 0)...}; } }; @@ -542,8 +680,9 @@ using extract_guard_t = typename exactly_one_t<is_call_guard, call_guard<>, Extr /// Check the number of named arguments at compile time template <typename... Extra, size_t named = constexpr_sum(std::is_base_of<arg, Extra>::value...), - size_t self = constexpr_sum(std::is_same<is_method, Extra>::value...)> + size_t self = constexpr_sum(std::is_same<is_method, Extra>::value...)> constexpr bool expected_num_args(size_t nargs, bool has_args, bool has_kwargs) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(nargs, has_args, has_kwargs); return named == 0 || (self + named + size_t(has_args) + size_t(has_kwargs)) == nargs; } diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index d803004a..b99ee8be 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -19,9 +19,11 @@ PYBIND11_NAMESPACE_BEGIN(detail) inline std::vector<ssize_t> c_strides(const std::vector<ssize_t> &shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector<ssize_t> strides(ndim, itemsize); - if (ndim > 0) - for (size_t i = ndim - 1; i > 0; --i) + if (ndim > 0) { + for (size_t i = ndim - 1; i > 0; --i) { strides[i - 1] = strides[i] * shape[i]; + } + } return strides; } @@ -29,11 +31,15 @@ inline std::vector<ssize_t> c_strides(const std::vector<ssize_t> &shape, ssize_t inline std::vector<ssize_t> f_strides(const std::vector<ssize_t> &shape, ssize_t itemsize) { auto ndim = shape.size(); std::vector<ssize_t> strides(ndim, itemsize); - for (size_t i = 1; i < ndim; ++i) + for (size_t i = 1; i < ndim; ++i) { strides[i] = strides[i - 1] * shape[i - 1]; + } return strides; } +template <typename T, typename SFINAE = void> +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -41,61 +47,89 @@ struct buffer_info { void *ptr = nullptr; // Pointer to the underlying storage ssize_t itemsize = 0; // Size of individual items in bytes ssize_t size = 0; // Total number of entries - std::string format; // For homogeneous buffers, this should be set to format_descriptor<T>::format() + std::string format; // For homogeneous buffers, this should be set to + // format_descriptor<T>::format() ssize_t ndim = 0; // Number of dimensions std::vector<ssize_t> shape; // Shape of the tensor (1 entry per dimension) - std::vector<ssize_t> strides; // Number of bytes between adjacent entries (for each per dimension) + std::vector<ssize_t> strides; // Number of bytes between adjacent entries + // (for each per dimension) bool readonly = false; // flag to indicate if the underlying storage may be written to buffer_info() = default; - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container<ssize_t> shape_in, detail::any_container<ssize_t> strides_in, bool readonly=false) - : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), - shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { - if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container<ssize_t> shape_in, + detail::any_container<ssize_t> strides_in, + bool readonly = false) + : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), + shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { + if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) { pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); - for (size_t i = 0; i < (size_t) ndim; ++i) + } + for (size_t i = 0; i < (size_t) ndim; ++i) { size *= shape[i]; + } } template <typename T> - buffer_info(T *ptr, detail::any_container<ssize_t> shape_in, detail::any_container<ssize_t> strides_in, bool readonly=false) - : buffer_info(private_ctr_tag(), ptr, sizeof(T), format_descriptor<T>::format(), static_cast<ssize_t>(shape_in->size()), std::move(shape_in), std::move(strides_in), readonly) { } - - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t size, bool readonly=false) - : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) { } + buffer_info(T *ptr, + detail::any_container<ssize_t> shape_in, + detail::any_container<ssize_t> strides_in, + bool readonly = false) + : buffer_info(private_ctr_tag(), + ptr, + sizeof(T), + format_descriptor<T>::format(), + static_cast<ssize_t>(shape_in->size()), + std::move(shape_in), + std::move(strides_in), + readonly) {} + + buffer_info(void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t size, + bool readonly = false) + : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) {} template <typename T> - buffer_info(T *ptr, ssize_t size, bool readonly=false) - : buffer_info(ptr, sizeof(T), format_descriptor<T>::format(), size, readonly) { } + buffer_info(T *ptr, ssize_t size, bool readonly = false) + : buffer_info(ptr, sizeof(T), format_descriptor<T>::format(), size, readonly) {} template <typename T> - buffer_info(const T *ptr, ssize_t size, bool readonly=true) - : buffer_info(const_cast<T*>(ptr), sizeof(T), format_descriptor<T>::format(), size, readonly) { } + buffer_info(const T *ptr, ssize_t size, bool readonly = true) + : buffer_info( + const_cast<T *>(ptr), sizeof(T), format_descriptor<T>::format(), size, readonly) {} explicit buffer_info(Py_buffer *view, bool ownview = true) - : buffer_info(view->buf, view->itemsize, view->format, view->ndim, + : buffer_info( + view->buf, + view->itemsize, + view->format, + view->ndim, {view->shape, view->shape + view->ndim}, /* Though buffer::request() requests PyBUF_STRIDES, ctypes objects * ignore this flag and return a view with NULL strides. * When strides are NULL, build them manually. */ view->strides - ? std::vector<ssize_t>(view->strides, view->strides + view->ndim) - : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), - view->readonly) { + ? std::vector<ssize_t>(view->strides, view->strides + view->ndim) + : detail::c_strides({view->shape, view->shape + view->ndim}, view->itemsize), + (view->readonly != 0)) { + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) this->m_view = view; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) this->ownview = ownview; } buffer_info(const buffer_info &) = delete; - buffer_info& operator=(const buffer_info &) = delete; + buffer_info &operator=(const buffer_info &) = delete; - buffer_info(buffer_info &&other) { - (*this) = std::move(other); - } + buffer_info(buffer_info &&other) noexcept { (*this) = std::move(other); } - buffer_info& operator=(buffer_info &&rhs) { + buffer_info &operator=(buffer_info &&rhs) noexcept { ptr = rhs.ptr; itemsize = rhs.itemsize; size = rhs.size; @@ -110,17 +144,39 @@ struct buffer_info { } ~buffer_info() { - if (m_view && ownview) { PyBuffer_Release(m_view); delete m_view; } + if (m_view && ownview) { + PyBuffer_Release(m_view); + delete m_view; + } } Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } -private: - struct private_ctr_tag { }; - buffer_info(private_ctr_tag, void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container<ssize_t> &&shape_in, detail::any_container<ssize_t> &&strides_in, bool readonly) - : buffer_info(ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) { } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to<int>(b)` and + // `buffer_info::item_type_is_equivalent_to<long>(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template <typename T> + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info<T>::compare(*this); + } + +private: + struct private_ctr_tag {}; + + buffer_info(private_ctr_tag, + void *ptr, + ssize_t itemsize, + const std::string &format, + ssize_t ndim, + detail::any_container<ssize_t> &&shape_in, + detail::any_container<ssize_t> &&strides_in, + bool readonly) + : buffer_info( + ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) {} Py_buffer *m_view = nullptr; bool ownview = false; @@ -128,17 +184,23 @@ private: PYBIND11_NAMESPACE_BEGIN(detail) -template <typename T, typename SFINAE = void> struct compare_buffer_info { - static bool compare(const buffer_info& b) { +template <typename T, typename SFINAE> +struct compare_buffer_info { + static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor<T>::format() && b.itemsize == (ssize_t) sizeof(T); } }; -template <typename T> struct compare_buffer_info<T, detail::enable_if_t<std::is_integral<T>::value>> { - static bool compare(const buffer_info& b) { - return (size_t) b.itemsize == sizeof(T) && (b.format == format_descriptor<T>::value || - ((sizeof(T) == sizeof(long)) && b.format == (std::is_unsigned<T>::value ? "L" : "l")) || - ((sizeof(T) == sizeof(size_t)) && b.format == (std::is_unsigned<T>::value ? "N" : "n"))); +template <typename T> +struct compare_buffer_info<T, detail::enable_if_t<std::is_integral<T>::value>> { + static bool compare(const buffer_info &b) { + return (size_t) b.itemsize == sizeof(T) + && (b.format == format_descriptor<T>::value + || ((sizeof(T) == sizeof(long)) + && b.format == (std::is_unsigned<T>::value ? "L" : "l")) + || ((sizeof(T) == sizeof(size_t)) + && b.format == (std::is_unsigned<T>::value ? "N" : "n"))); } }; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 0caccdb2..db393411 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -10,1020 +10,131 @@ #pragma once -#include "pytypes.h" -#include "detail/typeid.h" +#include "detail/common.h" #include "detail/descr.h" -#include "detail/internals.h" +#include "detail/type_caster_base.h" +#include "detail/typeid.h" +#include "pytypes.h" + #include <array> -#include <limits> +#include <cstring> +#include <functional> +#include <iosfwd> +#include <iterator> +#include <memory> +#include <string> #include <tuple> #include <type_traits> - -#if defined(PYBIND11_CPP17) -# if defined(__has_include) -# if __has_include(<string_view>) -# define PYBIND11_HAS_STRING_VIEW -# endif -# elif defined(_MSC_VER) -# define PYBIND11_HAS_STRING_VIEW -# endif -#endif -#ifdef PYBIND11_HAS_STRING_VIEW -#include <string_view> -#endif - -#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING -#endif +#include <utility> +#include <vector> PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) -PYBIND11_NAMESPACE_BEGIN(detail) - -/// A life support system for temporary objects created by `type_caster::load()`. -/// Adding a patient will keep it alive up until the enclosing function returns. -class loader_life_support { -public: - /// A new patient frame is created when a function is entered - loader_life_support() { - get_internals().loader_patient_stack.push_back(nullptr); - } - - /// ... and destroyed after it returns - ~loader_life_support() { - auto &stack = get_internals().loader_patient_stack; - if (stack.empty()) - pybind11_fail("loader_life_support: internal error"); - - auto ptr = stack.back(); - stack.pop_back(); - Py_CLEAR(ptr); - - // A heuristic to reduce the stack's capacity (e.g. after long recursive calls) - if (stack.capacity() > 16 && !stack.empty() && stack.capacity() / stack.size() > 2) - stack.shrink_to_fit(); - } - - /// This can only be used inside a pybind11-bound function, either by `argument_loader` - /// at argument preparation time or by `py::cast()` at execution time. - PYBIND11_NOINLINE static void add_patient(handle h) { - auto &stack = get_internals().loader_patient_stack; - if (stack.empty()) - throw cast_error("When called outside a bound function, py::cast() cannot " - "do Python -> C++ conversions which require the creation " - "of temporary values"); - - auto &list_ptr = stack.back(); - if (list_ptr == nullptr) { - list_ptr = PyList_New(1); - if (!list_ptr) - pybind11_fail("loader_life_support: error allocating list"); - PyList_SET_ITEM(list_ptr, 0, h.inc_ref().ptr()); - } else { - auto result = PyList_Append(list_ptr, h.ptr()); - if (result == -1) - pybind11_fail("loader_life_support: error adding patient"); - } - } -}; - -// Gets the cache entry for the given type, creating it if necessary. The return value is the pair -// returned by emplace, i.e. an iterator for the entry and a bool set to `true` if the entry was -// just created. -inline std::pair<decltype(internals::registered_types_py)::iterator, bool> all_type_info_get_cache(PyTypeObject *type); - -// Populates a just-created cache entry. -PYBIND11_NOINLINE inline void all_type_info_populate(PyTypeObject *t, std::vector<type_info *> &bases) { - std::vector<PyTypeObject *> check; - for (handle parent : reinterpret_borrow<tuple>(t->tp_bases)) - check.push_back((PyTypeObject *) parent.ptr()); - - auto const &type_dict = get_internals().registered_types_py; - for (size_t i = 0; i < check.size(); i++) { - auto type = check[i]; - // Ignore Python2 old-style class super type: - if (!PyType_Check((PyObject *) type)) continue; - - // Check `type` in the current set of registered python types: - auto it = type_dict.find(type); - if (it != type_dict.end()) { - // We found a cache entry for it, so it's either pybind-registered or has pre-computed - // pybind bases, but we have to make sure we haven't already seen the type(s) before: we - // want to follow Python/virtual C++ rules that there should only be one instance of a - // common base. - for (auto *tinfo : it->second) { - // NB: Could use a second set here, rather than doing a linear search, but since - // having a large number of immediate pybind11-registered types seems fairly - // unlikely, that probably isn't worthwhile. - bool found = false; - for (auto *known : bases) { - if (known == tinfo) { found = true; break; } - } - if (!found) bases.push_back(tinfo); - } - } - else if (type->tp_bases) { - // It's some python type, so keep follow its bases classes to look for one or more - // registered types - if (i + 1 == check.size()) { - // When we're at the end, we can pop off the current element to avoid growing - // `check` when adding just one base (which is typical--i.e. when there is no - // multiple inheritance) - check.pop_back(); - i--; - } - for (handle parent : reinterpret_borrow<tuple>(type->tp_bases)) - check.push_back((PyTypeObject *) parent.ptr()); - } - } -} - -/** - * Extracts vector of type_info pointers of pybind-registered roots of the given Python type. Will - * be just 1 pybind type for the Python type of a pybind-registered class, or for any Python-side - * derived class that uses single inheritance. Will contain as many types as required for a Python - * class that uses multiple inheritance to inherit (directly or indirectly) from multiple - * pybind-registered classes. Will be empty if neither the type nor any base classes are - * pybind-registered. - * - * The value is cached for the lifetime of the Python type. - */ -inline const std::vector<detail::type_info *> &all_type_info(PyTypeObject *type) { - auto ins = all_type_info_get_cache(type); - if (ins.second) - // New cache entry: populate it - all_type_info_populate(type, ins.first->second); - - return ins.first->second; -} - -/** - * Gets a single pybind11 type info for a python type. Returns nullptr if neither the type nor any - * ancestors are pybind11-registered. Throws an exception if there are multiple bases--use - * `all_type_info` instead if you want to support multiple bases. - */ -PYBIND11_NOINLINE inline detail::type_info* get_type_info(PyTypeObject *type) { - auto &bases = all_type_info(type); - if (bases.empty()) - return nullptr; - if (bases.size() > 1) - pybind11_fail("pybind11::detail::get_type_info: type has multiple pybind11-registered bases"); - return bases.front(); -} - -inline detail::type_info *get_local_type_info(const std::type_index &tp) { - auto &locals = registered_local_types_cpp(); - auto it = locals.find(tp); - if (it != locals.end()) - return it->second; - return nullptr; -} - -inline detail::type_info *get_global_type_info(const std::type_index &tp) { - auto &types = get_internals().registered_types_cpp; - auto it = types.find(tp); - if (it != types.end()) - return it->second; - return nullptr; -} - -/// Return the type info for a given C++ type; on lookup failure can either throw or return nullptr. -PYBIND11_NOINLINE inline detail::type_info *get_type_info(const std::type_index &tp, - bool throw_if_missing = false) { - if (auto ltype = get_local_type_info(tp)) - return ltype; - if (auto gtype = get_global_type_info(tp)) - return gtype; - - if (throw_if_missing) { - std::string tname = tp.name(); - detail::clean_type_id(tname); - pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + tname + "\""); - } - return nullptr; -} - -PYBIND11_NOINLINE inline handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { - detail::type_info *type_info = get_type_info(tp, throw_if_missing); - return handle(type_info ? ((PyObject *) type_info->type) : nullptr); -} - -struct value_and_holder { - instance *inst = nullptr; - size_t index = 0u; - const detail::type_info *type = nullptr; - void **vh = nullptr; - - // Main constructor for a found value/holder: - value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) : - inst{i}, index{index}, type{type}, - vh{inst->simple_layout ? inst->simple_value_holder : &inst->nonsimple.values_and_holders[vpos]} - {} - - // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) - value_and_holder() = default; - - // Used for past-the-end iterator - value_and_holder(size_t index) : index{index} {} - - template <typename V = void> V *&value_ptr() const { - return reinterpret_cast<V *&>(vh[0]); - } - // True if this `value_and_holder` has a non-null value pointer - explicit operator bool() const { return value_ptr(); } - - template <typename H> H &holder() const { - return reinterpret_cast<H &>(vh[1]); - } - bool holder_constructed() const { - return inst->simple_layout - ? inst->simple_holder_constructed - : inst->nonsimple.status[index] & instance::status_holder_constructed; - } - void set_holder_constructed(bool v = true) { - if (inst->simple_layout) - inst->simple_holder_constructed = v; - else if (v) - inst->nonsimple.status[index] |= instance::status_holder_constructed; - else - inst->nonsimple.status[index] &= (uint8_t) ~instance::status_holder_constructed; - } - bool instance_registered() const { - return inst->simple_layout - ? inst->simple_instance_registered - : inst->nonsimple.status[index] & instance::status_instance_registered; - } - void set_instance_registered(bool v = true) { - if (inst->simple_layout) - inst->simple_instance_registered = v; - else if (v) - inst->nonsimple.status[index] |= instance::status_instance_registered; - else - inst->nonsimple.status[index] &= (uint8_t) ~instance::status_instance_registered; - } -}; - -// Container for accessing and iterating over an instance's values/holders -struct values_and_holders { -private: - instance *inst; - using type_vec = std::vector<detail::type_info *>; - const type_vec &tinfo; - -public: - values_and_holders(instance *inst) : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} - - struct iterator { - private: - instance *inst = nullptr; - const type_vec *types = nullptr; - value_and_holder curr; - friend struct values_and_holders; - iterator(instance *inst, const type_vec *tinfo) - : inst{inst}, types{tinfo}, - curr(inst /* instance */, - types->empty() ? nullptr : (*types)[0] /* type info */, - 0, /* vpos: (non-simple types only): the first vptr comes first */ - 0 /* index */) - {} - // Past-the-end iterator: - iterator(size_t end) : curr(end) {} - public: - bool operator==(const iterator &other) const { return curr.index == other.curr.index; } - bool operator!=(const iterator &other) const { return curr.index != other.curr.index; } - iterator &operator++() { - if (!inst->simple_layout) - curr.vh += 1 + (*types)[curr.index]->holder_size_in_ptrs; - ++curr.index; - curr.type = curr.index < types->size() ? (*types)[curr.index] : nullptr; - return *this; - } - value_and_holder &operator*() { return curr; } - value_and_holder *operator->() { return &curr; } - }; - - iterator begin() { return iterator(inst, &tinfo); } - iterator end() { return iterator(tinfo.size()); } - - iterator find(const type_info *find_type) { - auto it = begin(), endit = end(); - while (it != endit && it->type != find_type) ++it; - return it; - } - - size_t size() { return tinfo.size(); } -}; - -/** - * Extracts C++ value and holder pointer references from an instance (which may contain multiple - * values/holders for python-side multiple inheritance) that match the given type. Throws an error - * if the given type (or ValueType, if omitted) is not a pybind11 base of the given instance. If - * `find_type` is omitted (or explicitly specified as nullptr) the first value/holder are returned, - * regardless of type (and the resulting .type will be nullptr). - * - * The returned object should be short-lived: in particular, it must not outlive the called-upon - * instance. - */ -PYBIND11_NOINLINE inline value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { - // Optimize common case: - if (!find_type || Py_TYPE(this) == find_type->type) - return value_and_holder(this, find_type, 0, 0); - - detail::values_and_holders vhs(this); - auto it = vhs.find(find_type); - if (it != vhs.end()) - return *it; - - if (!throw_if_missing) - return value_and_holder(); - -#if defined(NDEBUG) - pybind11_fail("pybind11::detail::instance::get_value_and_holder: " - "type is not a pybind11 base of the given instance " - "(compile in debug mode for type details)"); -#else - pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + - get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" + - get_fully_qualified_tp_name(Py_TYPE(this)) + "' instance"); -#endif -} - -PYBIND11_NOINLINE inline void instance::allocate_layout() { - auto &tinfo = all_type_info(Py_TYPE(this)); - - const size_t n_types = tinfo.size(); - - if (n_types == 0) - pybind11_fail("instance allocation failed: new instance has no pybind11-registered base types"); - - simple_layout = - n_types == 1 && tinfo.front()->holder_size_in_ptrs <= instance_simple_holder_in_ptrs(); - - // Simple path: no python-side multiple inheritance, and a small-enough holder - if (simple_layout) { - simple_value_holder[0] = nullptr; - simple_holder_constructed = false; - simple_instance_registered = false; - } - else { // multiple base types or a too-large holder - // Allocate space to hold: [v1*][h1][v2*][h2]...[bb...] where [vN*] is a value pointer, - // [hN] is the (uninitialized) holder instance for value N, and [bb...] is a set of bool - // values that tracks whether each associated holder has been initialized. Each [block] is - // padded, if necessary, to an integer multiple of sizeof(void *). - size_t space = 0; - for (auto t : tinfo) { - space += 1; // value pointer - space += t->holder_size_in_ptrs; // holder instance - } - size_t flags_at = space; - space += size_in_ptrs(n_types); // status bytes (holder_constructed and instance_registered) - - // Allocate space for flags, values, and holders, and initialize it to 0 (flags and values, - // in particular, need to be 0). Use Python's memory allocation functions: in Python 3.6 - // they default to using pymalloc, which is designed to be efficient for small allocations - // like the one we're doing here; in earlier versions (and for larger allocations) they are - // just wrappers around malloc. -#if PY_VERSION_HEX >= 0x03050000 - nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); - if (!nonsimple.values_and_holders) throw std::bad_alloc(); -#else - nonsimple.values_and_holders = (void **) PyMem_New(void *, space); - if (!nonsimple.values_and_holders) throw std::bad_alloc(); - std::memset(nonsimple.values_and_holders, 0, space * sizeof(void *)); -#endif - nonsimple.status = reinterpret_cast<uint8_t *>(&nonsimple.values_and_holders[flags_at]); - } - owned = true; -} - -PYBIND11_NOINLINE inline void instance::deallocate_layout() { - if (!simple_layout) - PyMem_Free(nonsimple.values_and_holders); -} - -PYBIND11_NOINLINE inline bool isinstance_generic(handle obj, const std::type_info &tp) { - handle type = detail::get_type_handle(tp, false); - if (!type) - return false; - return isinstance(obj, type); -} - -PYBIND11_NOINLINE inline std::string error_string() { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Unknown internal error occurred"); - return "Unknown internal error occurred"; - } - - error_scope scope; // Preserve error state - - std::string errorString; - if (scope.type) { - errorString += handle(scope.type).attr("__name__").cast<std::string>(); - errorString += ": "; - } - if (scope.value) - errorString += (std::string) str(scope.value); - - PyErr_NormalizeException(&scope.type, &scope.value, &scope.trace); - -#if PY_MAJOR_VERSION >= 3 - if (scope.trace != nullptr) - PyException_SetTraceback(scope.value, scope.trace); -#endif - -#if !defined(PYPY_VERSION) - if (scope.trace) { - auto *trace = (PyTracebackObject *) scope.trace; - - /* Get the deepest trace possible */ - while (trace->tb_next) - trace = trace->tb_next; - - PyFrameObject *frame = trace->tb_frame; - errorString += "\n\nAt:\n"; - while (frame) { - int lineno = PyFrame_GetLineNumber(frame); - errorString += - " " + handle(frame->f_code->co_filename).cast<std::string>() + - "(" + std::to_string(lineno) + "): " + - handle(frame->f_code->co_name).cast<std::string>() + "\n"; - frame = frame->f_back; - } - } -#endif - - return errorString; -} - -PYBIND11_NOINLINE inline handle get_object_handle(const void *ptr, const detail::type_info *type ) { - auto &instances = get_internals().registered_instances; - auto range = instances.equal_range(ptr); - for (auto it = range.first; it != range.second; ++it) { - for (const auto &vh : values_and_holders(it->second)) { - if (vh.type == type) - return handle((PyObject *) it->second); - } - } - return handle(); -} - -inline PyThreadState *get_thread_state_unchecked() { -#if defined(PYPY_VERSION) - return PyThreadState_GET(); -#elif PY_VERSION_HEX < 0x03000000 - return _PyThreadState_Current; -#elif PY_VERSION_HEX < 0x03050000 - return (PyThreadState*) _Py_atomic_load_relaxed(&_PyThreadState_Current); -#elif PY_VERSION_HEX < 0x03050200 - return (PyThreadState*) _PyThreadState_Current.value; -#else - return _PyThreadState_UncheckedGet(); -#endif -} - -// Forward declarations -inline void keep_alive_impl(handle nurse, handle patient); -inline PyObject *make_new_instance(PyTypeObject *type); - -class type_caster_generic { -public: - PYBIND11_NOINLINE type_caster_generic(const std::type_info &type_info) - : typeinfo(get_type_info(type_info)), cpptype(&type_info) { } - - type_caster_generic(const type_info *typeinfo) - : typeinfo(typeinfo), cpptype(typeinfo ? typeinfo->cpptype : nullptr) { } - - bool load(handle src, bool convert) { - return load_impl<type_caster_generic>(src, convert); - } - PYBIND11_NOINLINE static handle cast(const void *_src, return_value_policy policy, handle parent, - const detail::type_info *tinfo, - void *(*copy_constructor)(const void *), - void *(*move_constructor)(const void *), - const void *existing_holder = nullptr) { - if (!tinfo) // no type info: error will be set already - return handle(); - - void *src = const_cast<void *>(_src); - if (src == nullptr) - return none().release(); - - auto it_instances = get_internals().registered_instances.equal_range(src); - for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { - for (auto instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { - if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) - return handle((PyObject *) it_i->second).inc_ref(); - } - } - - auto inst = reinterpret_steal<object>(make_new_instance(tinfo->type)); - auto wrapper = reinterpret_cast<instance *>(inst.ptr()); - wrapper->owned = false; - void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); - - switch (policy) { - case return_value_policy::automatic: - case return_value_policy::take_ownership: - valueptr = src; - wrapper->owned = true; - break; - - case return_value_policy::automatic_reference: - case return_value_policy::reference: - valueptr = src; - wrapper->owned = false; - break; - - case return_value_policy::copy: - if (copy_constructor) - valueptr = copy_constructor(src); - else { -#if defined(NDEBUG) - throw cast_error("return_value_policy = copy, but type is " - "non-copyable! (compile in debug mode for details)"); -#else - std::string type_name(tinfo->cpptype->name()); - detail::clean_type_id(type_name); - throw cast_error("return_value_policy = copy, but type " + - type_name + " is non-copyable!"); -#endif - } - wrapper->owned = true; - break; - - case return_value_policy::move: - if (move_constructor) - valueptr = move_constructor(src); - else if (copy_constructor) - valueptr = copy_constructor(src); - else { -#if defined(NDEBUG) - throw cast_error("return_value_policy = move, but type is neither " - "movable nor copyable! " - "(compile in debug mode for details)"); -#else - std::string type_name(tinfo->cpptype->name()); - detail::clean_type_id(type_name); - throw cast_error("return_value_policy = move, but type " + - type_name + " is neither movable nor copyable!"); -#endif - } - wrapper->owned = true; - break; - - case return_value_policy::reference_internal: - valueptr = src; - wrapper->owned = false; - keep_alive_impl(inst, parent); - break; - - default: - throw cast_error("unhandled return_value_policy: should not happen!"); - } - - tinfo->init_instance(wrapper, existing_holder); - - return inst.release(); - } - - // Base methods for generic caster; there are overridden in copyable_holder_caster - void load_value(value_and_holder &&v_h) { - auto *&vptr = v_h.value_ptr(); - // Lazy allocation for unallocated values: - if (vptr == nullptr) { - auto *type = v_h.type ? v_h.type : typeinfo; - if (type->operator_new) { - vptr = type->operator_new(type->type_size); - } else { - #if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) - if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) - vptr = ::operator new(type->type_size, - std::align_val_t(type->type_align)); - else - #endif - vptr = ::operator new(type->type_size); - } - } - value = vptr; - } - bool try_implicit_casts(handle src, bool convert) { - for (auto &cast : typeinfo->implicit_casts) { - type_caster_generic sub_caster(*cast.first); - if (sub_caster.load(src, convert)) { - value = cast.second(sub_caster.value); - return true; - } - } - return false; - } - bool try_direct_conversions(handle src) { - for (auto &converter : *typeinfo->direct_conversions) { - if (converter(src.ptr(), value)) - return true; - } - return false; - } - void check_holder_compat() {} - - PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { - auto caster = type_caster_generic(ti); - if (caster.load(src, false)) - return caster.value; - return nullptr; - } - - /// Try to load with foreign typeinfo, if available. Used when there is no - /// native typeinfo, or when the native one wasn't able to produce a value. - PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { - constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; - const auto pytype = type::handle_of(src); - if (!hasattr(pytype, local_key)) - return false; - - type_info *foreign_typeinfo = reinterpret_borrow<capsule>(getattr(pytype, local_key)); - // Only consider this foreign loader if actually foreign and is a loader of the correct cpp type - if (foreign_typeinfo->module_local_load == &local_load - || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) - return false; - - if (auto result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { - value = result; - return true; - } - return false; - } - - // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant - // bits of code between here and copyable_holder_caster where the two classes need different - // logic (without having to resort to virtual inheritance). - template <typename ThisT> - PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { - if (!src) return false; - if (!typeinfo) return try_load_foreign_module_local(src); - if (src.is_none()) { - // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; - value = nullptr; - return true; - } - - auto &this_ = static_cast<ThisT &>(*this); - this_.check_holder_compat(); - - PyTypeObject *srctype = Py_TYPE(src.ptr()); - - // Case 1: If src is an exact type match for the target type then we can reinterpret_cast - // the instance's value pointer to the target type: - if (srctype == typeinfo->type) { - this_.load_value(reinterpret_cast<instance *>(src.ptr())->get_value_and_holder()); - return true; - } - // Case 2: We have a derived class - else if (PyType_IsSubtype(srctype, typeinfo->type)) { - auto &bases = all_type_info(srctype); - bool no_cpp_mi = typeinfo->simple_type; - - // Case 2a: the python type is a Python-inherited derived class that inherits from just - // one simple (no MI) pybind11 class, or is an exact match, so the C++ instance is of - // the right type and we can use reinterpret_cast. - // (This is essentially the same as case 2b, but because not using multiple inheritance - // is extremely common, we handle it specially to avoid the loop iterator and type - // pointer lookup overhead) - if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { - this_.load_value(reinterpret_cast<instance *>(src.ptr())->get_value_and_holder()); - return true; - } - // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see if - // we can find an exact match (or, for a simple C++ type, an inherited match); if so, we - // can safely reinterpret_cast to the relevant pointer. - else if (bases.size() > 1) { - for (auto base : bases) { - if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) : base->type == typeinfo->type) { - this_.load_value(reinterpret_cast<instance *>(src.ptr())->get_value_and_holder(base)); - return true; - } - } - } - - // Case 2c: C++ multiple inheritance is involved and we couldn't find an exact type match - // in the registered bases, above, so try implicit casting (needed for proper C++ casting - // when MI is involved). - if (this_.try_implicit_casts(src, convert)) - return true; - } - - // Perform an implicit conversion - if (convert) { - for (auto &converter : typeinfo->implicit_conversions) { - auto temp = reinterpret_steal<object>(converter(src.ptr(), typeinfo->type)); - if (load_impl<ThisT>(temp, false)) { - loader_life_support::add_patient(temp); - return true; - } - } - if (this_.try_direct_conversions(src)) - return true; - } - - // Failed to match local typeinfo. Try again with global. - if (typeinfo->module_local) { - if (auto gtype = get_global_type_info(*typeinfo->cpptype)) { - typeinfo = gtype; - return load(src, false); - } - } - - // Global typeinfo has precedence over foreign module_local - return try_load_foreign_module_local(src); - } - - - // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast - // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair - // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). - PYBIND11_NOINLINE static std::pair<const void *, const type_info *> src_and_type( - const void *src, const std::type_info &cast_type, const std::type_info *rtti_type = nullptr) { - if (auto *tpi = get_type_info(cast_type)) - return {src, const_cast<const type_info *>(tpi)}; - - // Not found, set error: - std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); - detail::clean_type_id(tname); - std::string msg = "Unregistered type : " + tname; - PyErr_SetString(PyExc_TypeError, msg.c_str()); - return {nullptr, nullptr}; - } - - const type_info *typeinfo = nullptr; - const std::type_info *cpptype = nullptr; - void *value = nullptr; -}; - -/** - * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster - * needs to provide `operator T*()` and `operator T&()` operators. - * - * If the type supports moving the value away via an `operator T&&() &&` method, it should use - * `movable_cast_op_type` instead. - */ -template <typename T> -using cast_op_type = - conditional_t<std::is_pointer<remove_reference_t<T>>::value, - typename std::add_pointer<intrinsic_t<T>>::type, - typename std::add_lvalue_reference<intrinsic_t<T>>::type>; - -/** - * Determine suitable casting operator for a type caster with a movable value. Such a type caster - * needs to provide `operator T*()`, `operator T&()`, and `operator T&&() &&`. The latter will be - * called in appropriate contexts where the value can be moved rather than copied. - * - * These operator are automatically provided when using the PYBIND11_TYPE_CASTER macro. - */ -template <typename T> -using movable_cast_op_type = - conditional_t<std::is_pointer<typename std::remove_reference<T>::type>::value, - typename std::add_pointer<intrinsic_t<T>>::type, - conditional_t<std::is_rvalue_reference<T>::value, - typename std::add_rvalue_reference<intrinsic_t<T>>::type, - typename std::add_lvalue_reference<intrinsic_t<T>>::type>>; - -// std::is_copy_constructible isn't quite enough: it lets std::vector<T> (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template <typename T, typename SFINAE = void> struct is_copy_constructible : std::is_copy_constructible<T> {}; - -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. -template <typename Container> struct is_copy_constructible<Container, enable_if_t<all_of< - std::is_copy_constructible<Container>, - std::is_same<typename Container::value_type &, typename Container::reference>, - // Avoid infinite recursion - negation<std::is_same<Container, typename Container::value_type>> - >::value>> : is_copy_constructible<typename Container::value_type> {}; - -// Likewise for std::pair -// (after C++17 it is mandatory that the copy constructor not exist when the two types aren't themselves -// copy constructible, but this can not be relied upon when T1 or T2 are themselves containers). -template <typename T1, typename T2> struct is_copy_constructible<std::pair<T1, T2>> - : all_of<is_copy_constructible<T1>, is_copy_constructible<T2>> {}; - -// The same problems arise with std::is_copy_assignable, so we use the same workaround. -template <typename T, typename SFINAE = void> struct is_copy_assignable : std::is_copy_assignable<T> {}; -template <typename Container> struct is_copy_assignable<Container, enable_if_t<all_of< - std::is_copy_assignable<Container>, - std::is_same<typename Container::value_type &, typename Container::reference> - >::value>> : is_copy_assignable<typename Container::value_type> {}; -template <typename T1, typename T2> struct is_copy_assignable<std::pair<T1, T2>> - : all_of<is_copy_assignable<T1>, is_copy_assignable<T2>> {}; - -PYBIND11_NAMESPACE_END(detail) - -// polymorphic_type_hook<itype>::get(src, tinfo) determines whether the object pointed -// to by `src` actually is an instance of some class derived from `itype`. -// If so, it sets `tinfo` to point to the std::type_info representing that derived -// type, and returns a pointer to the start of the most-derived object of that type -// (in which `src` is a subobject; this will be the same address as `src` in most -// single inheritance cases). If not, or if `src` is nullptr, it simply returns `src` -// and leaves `tinfo` at its default value of nullptr. -// -// The default polymorphic_type_hook just returns src. A specialization for polymorphic -// types determines the runtime type of the passed object and adjusts the this-pointer -// appropriately via dynamic_cast<void*>. This is what enables a C++ Animal* to appear -// to Python as a Dog (if Dog inherits from Animal, Animal is polymorphic, Dog is -// registered with pybind11, and this Animal is in fact a Dog). -// -// You may specialize polymorphic_type_hook yourself for types that want to appear -// polymorphic to Python but do not use C++ RTTI. (This is a not uncommon pattern -// in performance-sensitive applications, used most notably in LLVM.) -// -// polymorphic_type_hook_base allows users to specialize polymorphic_type_hook with -// std::enable_if. User provided specializations will always have higher priority than -// the default implementation and specialization provided in polymorphic_type_hook_base. -template <typename itype, typename SFINAE = void> -struct polymorphic_type_hook_base -{ - static const void *get(const itype *src, const std::type_info*&) { return src; } -}; -template <typename itype> -struct polymorphic_type_hook_base<itype, detail::enable_if_t<std::is_polymorphic<itype>::value>> -{ - static const void *get(const itype *src, const std::type_info*& type) { - type = src ? &typeid(*src) : nullptr; - return dynamic_cast<const void*>(src); - } -}; -template <typename itype, typename SFINAE = void> -struct polymorphic_type_hook : public polymorphic_type_hook_base<itype> {}; +PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) -/// Generic type caster for objects stored on the heap -template <typename type> class type_caster_base : public type_caster_generic { - using itype = intrinsic_t<type>; - -public: - static constexpr auto name = _<type>(); - - type_caster_base() : type_caster_base(typeid(type)) { } - explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) { } - - static handle cast(const itype &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) - policy = return_value_policy::copy; - return cast(&src, policy, parent); - } - - static handle cast(itype &&src, return_value_policy, handle parent) { - return cast(&src, return_value_policy::move, parent); - } - - // Returns a (pointer, type_info) pair taking care of necessary type lookup for a - // polymorphic type (using RTTI by default, but can be overridden by specializing - // polymorphic_type_hook). If the instance isn't derived, returns the base version. - static std::pair<const void *, const type_info *> src_and_type(const itype *src) { - auto &cast_type = typeid(itype); - const std::type_info *instance_type = nullptr; - const void *vsrc = polymorphic_type_hook<itype>::get(src, instance_type); - if (instance_type && !same_type(cast_type, *instance_type)) { - // This is a base pointer to a derived type. If the derived type is registered - // with pybind11, we want to make the full derived object available. - // In the typical case where itype is polymorphic, we get the correct - // derived pointer (which may be != base pointer) by a dynamic_cast to - // most derived type. If itype is not polymorphic, we won't get here - // except via a user-provided specialization of polymorphic_type_hook, - // and the user has promised that no this-pointer adjustment is - // required in that case, so it's OK to use static_cast. - if (const auto *tpi = get_type_info(*instance_type)) - return {vsrc, tpi}; - } - // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, so - // don't do a cast - return type_caster_generic::src_and_type(src, cast_type, instance_type); - } - - static handle cast(const itype *src, return_value_policy policy, handle parent) { - auto st = src_and_type(src); - return type_caster_generic::cast( - st.first, policy, parent, st.second, - make_copy_constructor(src), make_move_constructor(src)); - } - - static handle cast_holder(const itype *src, const void *holder) { - auto st = src_and_type(src); - return type_caster_generic::cast( - st.first, return_value_policy::take_ownership, {}, st.second, - nullptr, nullptr, holder); - } - - template <typename T> using cast_op_type = detail::cast_op_type<T>; - - operator itype*() { return (type *) value; } - operator itype&() { if (!value) throw reference_cast_error(); return *((itype *) value); } - -protected: - using Constructor = void *(*)(const void *); - - /* Only enabled when the types are {copy,move}-constructible *and* when the type - does not have a private operator new implementation. */ - template <typename T, typename = enable_if_t<is_copy_constructible<T>::value>> - static auto make_copy_constructor(const T *x) -> decltype(new T(*x), Constructor{}) { - return [](const void *arg) -> void * { - return new T(*reinterpret_cast<const T *>(arg)); - }; - } - - template <typename T, typename = enable_if_t<std::is_move_constructible<T>::value>> - static auto make_move_constructor(const T *x) -> decltype(new T(std::move(*const_cast<T *>(x))), Constructor{}) { - return [](const void *arg) -> void * { - return new T(std::move(*const_cast<T *>(reinterpret_cast<const T *>(arg)))); - }; - } - - static Constructor make_copy_constructor(...) { return nullptr; } - static Constructor make_move_constructor(...) { return nullptr; } -}; - -template <typename type, typename SFINAE = void> class type_caster : public type_caster_base<type> { }; -template <typename type> using make_caster = type_caster<intrinsic_t<type>>; +template <typename type, typename SFINAE = void> +class type_caster : public type_caster_base<type> {}; +template <typename type> +using make_caster = type_caster<intrinsic_t<type>>; // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T -template <typename T> typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) { +template <typename T> +typename make_caster<T>::template cast_op_type<T> cast_op(make_caster<T> &caster) { return caster.operator typename make_caster<T>::template cast_op_type<T>(); } -template <typename T> typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type> +template <typename T> +typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type> cast_op(make_caster<T> &&caster) { - return std::move(caster).operator - typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>(); + return std::move(caster).operator typename make_caster<T>:: + template cast_op_type<typename std::add_rvalue_reference<T>::type>(); } -template <typename type> class type_caster<std::reference_wrapper<type>> { +template <typename type> +class type_caster<std::reference_wrapper<type>> { private: using caster_t = make_caster<type>; caster_t subcaster; - using reference_t = type&; - using subcaster_cast_op_type = - typename caster_t::template cast_op_type<reference_t>; - - static_assert(std::is_same<typename std::remove_const<type>::type &, subcaster_cast_op_type>::value || - std::is_same<reference_t, subcaster_cast_op_type>::value, - "std::reference_wrapper<T> caster requires T to have a caster with an " - "`operator T &()` or `operator const T &()`"); + using reference_t = type &; + using subcaster_cast_op_type = typename caster_t::template cast_op_type<reference_t>; + + static_assert( + std::is_same<typename std::remove_const<type>::type &, subcaster_cast_op_type>::value + || std::is_same<reference_t, subcaster_cast_op_type>::value, + "std::reference_wrapper<T> caster requires T to have a caster with an " + "`operator T &()` or `operator const T &()`"); + public: bool load(handle src, bool convert) { return subcaster.load(src, convert); } static constexpr auto name = caster_t::name; - static handle cast(const std::reference_wrapper<type> &src, return_value_policy policy, handle parent) { + static handle + cast(const std::reference_wrapper<type> &src, return_value_policy policy, handle parent) { // It is definitely wrong to take ownership of this pointer, so mask that rvp - if (policy == return_value_policy::take_ownership || policy == return_value_policy::automatic) + if (policy == return_value_policy::take_ownership + || policy == return_value_policy::automatic) { policy = return_value_policy::automatic_reference; + } return caster_t::cast(&src.get(), policy, parent); } - template <typename T> using cast_op_type = std::reference_wrapper<type>; - operator std::reference_wrapper<type>() { return cast_op<type &>(subcaster); } + template <typename T> + using cast_op_type = std::reference_wrapper<type>; + explicit operator std::reference_wrapper<type>() { return cast_op<type &>(subcaster); } }; -#define PYBIND11_TYPE_CASTER(type, py_name) \ - protected: \ - type value; \ - public: \ - static constexpr auto name = py_name; \ - template <typename T_, enable_if_t<std::is_same<type, remove_cv_t<T_>>::value, int> = 0> \ - static handle cast(T_ *src, return_value_policy policy, handle parent) { \ - if (!src) return none().release(); \ - if (policy == return_value_policy::take_ownership) { \ - auto h = cast(std::move(*src), policy, parent); delete src; return h; \ - } else { \ - return cast(*src, policy, parent); \ - } \ - } \ - operator type*() { return &value; } \ - operator type&() { return value; } \ - operator type&&() && { return std::move(value); } \ - template <typename T_> using cast_op_type = pybind11::detail::movable_cast_op_type<T_> - - -template <typename CharT> using is_std_char_type = any_of< - std::is_same<CharT, char>, /* std::string */ +#define PYBIND11_TYPE_CASTER(type, py_name) \ +protected: \ + type value; \ + \ +public: \ + static constexpr auto name = py_name; \ + template <typename T_, \ + ::pybind11::detail::enable_if_t< \ + std::is_same<type, ::pybind11::detail::remove_cv_t<T_>>::value, \ + int> \ + = 0> \ + static ::pybind11::handle cast( \ + T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ + if (!src) \ + return ::pybind11::none().release(); \ + if (policy == ::pybind11::return_value_policy::take_ownership) { \ + auto h = cast(std::move(*src), policy, parent); \ + delete src; \ + return h; \ + } \ + return cast(*src, policy, parent); \ + } \ + operator type *() { return &value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator type &() { return value; } /* NOLINT(bugprone-macro-parentheses) */ \ + operator type &&() && { return std::move(value); } /* NOLINT(bugprone-macro-parentheses) */ \ + template <typename T_> \ + using cast_op_type = ::pybind11::detail::movable_cast_op_type<T_> + +template <typename CharT> +using is_std_char_type = any_of<std::is_same<CharT, char>, /* std::string */ #if defined(PYBIND11_HAS_U8STRING) - std::is_same<CharT, char8_t>, /* std::u8string */ + std::is_same<CharT, char8_t>, /* std::u8string */ #endif - std::is_same<CharT, char16_t>, /* std::u16string */ - std::is_same<CharT, char32_t>, /* std::u32string */ - std::is_same<CharT, wchar_t> /* std::wstring */ ->; - + std::is_same<CharT, char16_t>, /* std::u16string */ + std::is_same<CharT, char32_t>, /* std::u32string */ + std::is_same<CharT, wchar_t> /* std::wstring */ + >; template <typename T> struct type_caster<T, enable_if_t<std::is_arithmetic<T>::value && !is_std_char_type<T>::value>> { using _py_type_0 = conditional_t<sizeof(T) <= sizeof(long), long, long long>; - using _py_type_1 = conditional_t<std::is_signed<T>::value, _py_type_0, typename std::make_unsigned<_py_type_0>::type>; + using _py_type_1 = conditional_t<std::is_signed<T>::value, + _py_type_0, + typename std::make_unsigned<_py_type_0>::type>; using py_type = conditional_t<std::is_floating_point<T>::value, double, _py_type_1>; -public: +public: bool load(handle src, bool convert) { py_type py_value; - if (!src) + if (!src) { return false; + } #if !defined(PYPY_VERSION) auto index_check = [](PyObject *o) { return PyIndex_Check(o); }; @@ -1034,26 +145,26 @@ public: #endif if (std::is_floating_point<T>::value) { - if (convert || PyFloat_Check(src.ptr())) + if (convert || PyFloat_Check(src.ptr())) { py_value = (py_type) PyFloat_AsDouble(src.ptr()); - else + } else { return false; - } else if (PyFloat_Check(src.ptr())) { - return false; - } else if (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr())) { + } + } else if (PyFloat_Check(src.ptr()) + || (!convert && !PYBIND11_LONG_CHECK(src.ptr()) && !index_check(src.ptr()))) { return false; } else { handle src_or_index = src; -#if PY_VERSION_HEX < 0x03080000 + // PyPy: 7.3.7's 3.8 does not implement PyLong_*'s __index__ calls. +#if PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) object index; - if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) + if (!PYBIND11_LONG_CHECK(src.ptr())) { // So: index_check(src.ptr()) index = reinterpret_steal<object>(PyNumber_Index(src.ptr())); if (!index) { PyErr_Clear(); if (!convert) return false; - } - else { + } else { src_or_index = index; } } @@ -1062,8 +173,8 @@ public: py_value = as_unsigned<py_type>(src_or_index.ptr()); } else { // signed integer: py_value = sizeof(T) <= sizeof(long) - ? (py_type) PyLong_AsLong(src_or_index.ptr()) - : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); + ? (py_type) PyLong_AsLong(src_or_index.ptr()) + : (py_type) PYBIND11_LONG_AS_LONGLONG(src_or_index.ptr()); } } @@ -1072,12 +183,14 @@ public: // Check to see if the conversion is valid (integers should match exactly) // Signed/unsigned checks happen elsewhere - if (py_err || (std::is_integral<T>::value && sizeof(py_type) != sizeof(T) && py_value != (py_type) (T) py_value)) { + if (py_err + || (std::is_integral<T>::value && sizeof(py_type) != sizeof(T) + && py_value != (py_type) (T) py_value)) { PyErr_Clear(); - if (py_err && convert && PyNumber_Check(src.ptr())) { + if (py_err && convert && (PyNumber_Check(src.ptr()) != 0)) { auto tmp = reinterpret_steal<object>(std::is_floating_point<T>::value - ? PyNumber_Float(src.ptr()) - : PyNumber_Long(src.ptr())); + ? PyNumber_Float(src.ptr()) + : PyNumber_Long(src.ptr())); PyErr_Clear(); return load(tmp, false); } @@ -1088,62 +201,75 @@ public: return true; } - template<typename U = T> + template <typename U = T> static typename std::enable_if<std::is_floating_point<U>::value, handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PyFloat_FromDouble((double) src); } - template<typename U = T> - static typename std::enable_if<!std::is_floating_point<U>::value && std::is_signed<U>::value && (sizeof(U) <= sizeof(long)), handle>::type + template <typename U = T> + static typename std::enable_if<!std::is_floating_point<U>::value && std::is_signed<U>::value + && (sizeof(U) <= sizeof(long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PYBIND11_LONG_FROM_SIGNED((long) src); } - template<typename U = T> - static typename std::enable_if<!std::is_floating_point<U>::value && std::is_unsigned<U>::value && (sizeof(U) <= sizeof(unsigned long)), handle>::type + template <typename U = T> + static typename std::enable_if<!std::is_floating_point<U>::value && std::is_unsigned<U>::value + && (sizeof(U) <= sizeof(unsigned long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PYBIND11_LONG_FROM_UNSIGNED((unsigned long) src); } - template<typename U = T> - static typename std::enable_if<!std::is_floating_point<U>::value && std::is_signed<U>::value && (sizeof(U) > sizeof(long)), handle>::type + template <typename U = T> + static typename std::enable_if<!std::is_floating_point<U>::value && std::is_signed<U>::value + && (sizeof(U) > sizeof(long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PyLong_FromLongLong((long long) src); } - template<typename U = T> - static typename std::enable_if<!std::is_floating_point<U>::value && std::is_unsigned<U>::value && (sizeof(U) > sizeof(unsigned long)), handle>::type + template <typename U = T> + static typename std::enable_if<!std::is_floating_point<U>::value && std::is_unsigned<U>::value + && (sizeof(U) > sizeof(unsigned long)), + handle>::type cast(U src, return_value_policy /* policy */, handle /* parent */) { return PyLong_FromUnsignedLongLong((unsigned long long) src); } - PYBIND11_TYPE_CASTER(T, _<std::is_integral<T>::value>("int", "float")); + PYBIND11_TYPE_CASTER(T, const_name<std::is_integral<T>::value>("int", "float")); }; -template<typename T> struct void_caster { +template <typename T> +struct void_caster { public: bool load(handle src, bool) { - if (src && src.is_none()) + if (src && src.is_none()) { return true; + } return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } - PYBIND11_TYPE_CASTER(T, _("None")); + PYBIND11_TYPE_CASTER(T, const_name("None")); }; -template <> class type_caster<void_type> : public void_caster<void_type> {}; +template <> +class type_caster<void_type> : public void_caster<void_type> {}; -template <> class type_caster<void> : public type_caster<void_type> { +template <> +class type_caster<void> : public type_caster<void_type> { public: using type_caster<void_type>::cast; bool load(handle h, bool) { if (!h) { return false; - } else if (h.is_none()) { + } + if (h.is_none()) { value = nullptr; return true; } @@ -1155,7 +281,7 @@ public: } /* Check if this is a C++ type */ - auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); + const auto &bases = all_type_info((PyTypeObject *) type::handle_of(h).ptr()); if (bases.size() == 1) { // Only allowing loading from a single-value type value = values_and_holders(reinterpret_cast<instance *>(h.ptr())).begin()->value_ptr(); return true; @@ -1166,191 +292,247 @@ public: } static handle cast(const void *ptr, return_value_policy /* policy */, handle /* parent */) { - if (ptr) + if (ptr) { return capsule(ptr).release(); - else - return none().inc_ref(); + } + return none().release(); } - template <typename T> using cast_op_type = void*&; - operator void *&() { return value; } - static constexpr auto name = _("capsule"); + template <typename T> + using cast_op_type = void *&; + explicit operator void *&() { return value; } + static constexpr auto name = const_name("capsule"); + private: void *value = nullptr; }; -template <> class type_caster<std::nullptr_t> : public void_caster<std::nullptr_t> { }; +template <> +class type_caster<std::nullptr_t> : public void_caster<std::nullptr_t> {}; -template <> class type_caster<bool> { +template <> +class type_caster<bool> { public: bool load(handle src, bool convert) { - if (!src) return false; - else if (src.ptr() == Py_True) { value = true; return true; } - else if (src.ptr() == Py_False) { value = false; return true; } - else if (convert || !strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name)) { + if (!src) { + return false; + } + if (src.ptr() == Py_True) { + value = true; + return true; + } + if (src.ptr() == Py_False) { + value = false; + return true; + } + if (convert || (std::strcmp("numpy.bool_", Py_TYPE(src.ptr())->tp_name) == 0)) { // (allow non-implicit conversion for numpy booleans) Py_ssize_t res = -1; if (src.is_none()) { - res = 0; // None is implicitly converted to False + res = 0; // None is implicitly converted to False } - #if defined(PYPY_VERSION) - // On PyPy, check that "__bool__" (or "__nonzero__" on Python 2.7) attr exists +#if defined(PYPY_VERSION) + // On PyPy, check that "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } - #else +#else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. - else if (auto tp_as_number = src.ptr()->ob_type->tp_as_number) { + else if (auto *tp_as_number = src.ptr()->ob_type->tp_as_number) { if (PYBIND11_NB_BOOL(tp_as_number)) { res = (*PYBIND11_NB_BOOL(tp_as_number))(src.ptr()); } } - #endif +#endif if (res == 0 || res == 1) { - value = (bool) res; + value = (res != 0); return true; - } else { - PyErr_Clear(); } + PyErr_Clear(); } return false; } static handle cast(bool src, return_value_policy /* policy */, handle /* parent */) { return handle(src ? Py_True : Py_False).inc_ref(); } - PYBIND11_TYPE_CASTER(bool, _("bool")); + PYBIND11_TYPE_CASTER(bool, const_name("bool")); }; // Helper class for UTF-{8,16,32} C++ stl strings: -template <typename StringType, bool IsView = false> struct string_caster { +template <typename StringType, bool IsView = false> +struct string_caster { using CharT = typename StringType::value_type; // Simplify life by being able to assume standard char sizes (the standard only guarantees // minimums, but Python requires exact sizes) - static_assert(!std::is_same<CharT, char>::value || sizeof(CharT) == 1, "Unsupported char size != 1"); + static_assert(!std::is_same<CharT, char>::value || sizeof(CharT) == 1, + "Unsupported char size != 1"); #if defined(PYBIND11_HAS_U8STRING) - static_assert(!std::is_same<CharT, char8_t>::value || sizeof(CharT) == 1, "Unsupported char8_t size != 1"); + static_assert(!std::is_same<CharT, char8_t>::value || sizeof(CharT) == 1, + "Unsupported char8_t size != 1"); #endif - static_assert(!std::is_same<CharT, char16_t>::value || sizeof(CharT) == 2, "Unsupported char16_t size != 2"); - static_assert(!std::is_same<CharT, char32_t>::value || sizeof(CharT) == 4, "Unsupported char32_t size != 4"); + static_assert(!std::is_same<CharT, char16_t>::value || sizeof(CharT) == 2, + "Unsupported char16_t size != 2"); + static_assert(!std::is_same<CharT, char32_t>::value || sizeof(CharT) == 4, + "Unsupported char32_t size != 4"); // wchar_t can be either 16 bits (Windows) or 32 (everywhere else) static_assert(!std::is_same<CharT, wchar_t>::value || sizeof(CharT) == 2 || sizeof(CharT) == 4, - "Unsupported wchar_t size != 2/4"); + "Unsupported wchar_t size != 2/4"); static constexpr size_t UTF_N = 8 * sizeof(CharT); bool load(handle src, bool) { -#if PY_MAJOR_VERSION < 3 - object temp; -#endif handle load_src = src; if (!src) { return false; - } else if (!PyUnicode_Check(load_src.ptr())) { -#if PY_MAJOR_VERSION >= 3 - return load_bytes(load_src); -#else - if (std::is_same<CharT, char>::value) { - return load_bytes(load_src); - } + } + if (!PyUnicode_Check(load_src.ptr())) { + return load_raw(load_src); + } - // The below is a guaranteed failure in Python 3 when PyUnicode_Check returns false - if (!PYBIND11_BYTES_CHECK(load_src.ptr())) + // For UTF-8 we avoid the need for a temporary `bytes` object by using + // `PyUnicode_AsUTF8AndSize`. + if (UTF_N == 8) { + Py_ssize_t size = -1; + const auto *buffer + = reinterpret_cast<const CharT *>(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); + if (!buffer) { + PyErr_Clear(); return false; - - temp = reinterpret_steal<object>(PyUnicode_FromObject(load_src.ptr())); - if (!temp) { PyErr_Clear(); return false; } - load_src = temp; -#endif + } + value = StringType(buffer, static_cast<size_t>(size)); + return true; } - auto utfNbytes = reinterpret_steal<object>(PyUnicode_AsEncodedString( - load_src.ptr(), UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr)); - if (!utfNbytes) { PyErr_Clear(); return false; } + auto utfNbytes + = reinterpret_steal<object>(PyUnicode_AsEncodedString(load_src.ptr(), + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr)); + if (!utfNbytes) { + PyErr_Clear(); + return false; + } - const auto *buffer = reinterpret_cast<const CharT *>(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); + const auto *buffer + = reinterpret_cast<const CharT *>(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); - if (UTF_N > 8) { buffer++; length--; } // Skip BOM for UTF-16/32 + // Skip BOM for UTF-16/32 + if (UTF_N > 8) { + buffer++; + length--; + } value = StringType(buffer, length); // If we're loading a string_view we need to keep the encoded Python object alive: - if (IsView) + if (IsView) { loader_life_support::add_patient(utfNbytes); + } return true; } - static handle cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { + static handle + cast(const StringType &src, return_value_policy /* policy */, handle /* parent */) { const char *buffer = reinterpret_cast<const char *>(src.data()); auto nbytes = ssize_t(src.size() * sizeof(CharT)); handle s = decode_utfN(buffer, nbytes); - if (!s) throw error_already_set(); + if (!s) { + throw error_already_set(); + } return s; } - PYBIND11_TYPE_CASTER(StringType, _(PYBIND11_STRING_NAME)); + PYBIND11_TYPE_CASTER(StringType, const_name(PYBIND11_STRING_NAME)); private: static handle decode_utfN(const char *buffer, ssize_t nbytes) { #if !defined(PYPY_VERSION) - return - UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) : - UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) : - PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); + return UTF_N == 8 ? PyUnicode_DecodeUTF8(buffer, nbytes, nullptr) + : UTF_N == 16 ? PyUnicode_DecodeUTF16(buffer, nbytes, nullptr, nullptr) + : PyUnicode_DecodeUTF32(buffer, nbytes, nullptr, nullptr); #else - // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as well), - // so bypass the whole thing by just passing the encoding as a string value, which works properly: - return PyUnicode_Decode(buffer, nbytes, UTF_N == 8 ? "utf-8" : UTF_N == 16 ? "utf-16" : "utf-32", nullptr); + // PyPy segfaults when on PyUnicode_DecodeUTF16 (and possibly on PyUnicode_DecodeUTF32 as + // well), so bypass the whole thing by just passing the encoding as a string value, which + // works properly: + return PyUnicode_Decode(buffer, + nbytes, + UTF_N == 8 ? "utf-8" + : UTF_N == 16 ? "utf-16" + : "utf-32", + nullptr); #endif } - // When loading into a std::string or char*, accept a bytes object as-is (i.e. + // When loading into a std::string or char*, accept a bytes/bytearray object as-is (i.e. // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. // which supports loading a unicode from a str, doesn't take this path. template <typename C = CharT> - bool load_bytes(enable_if_t<std::is_same<C, char>::value, handle> src) { + bool load_raw(enable_if_t<std::is_same<C, char>::value, handle> src) { if (PYBIND11_BYTES_CHECK(src.ptr())) { - // We were passed a Python 3 raw bytes; accept it into a std::string or char* + // We were passed raw bytes; accept it into a std::string or char* // without any encoding attempt. const char *bytes = PYBIND11_BYTES_AS_STRING(src.ptr()); - if (bytes) { - value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); - return true; + if (!bytes) { + pybind11_fail("Unexpected PYBIND11_BYTES_AS_STRING() failure."); + } + value = StringType(bytes, (size_t) PYBIND11_BYTES_SIZE(src.ptr())); + return true; + } + if (PyByteArray_Check(src.ptr())) { + // We were passed a bytearray; accept it into a std::string or char* + // without any encoding attempt. + const char *bytearray = PyByteArray_AsString(src.ptr()); + if (!bytearray) { + pybind11_fail("Unexpected PyByteArray_AsString() failure."); } + value = StringType(bytearray, (size_t) PyByteArray_Size(src.ptr())); + return true; } return false; } template <typename C = CharT> - bool load_bytes(enable_if_t<!std::is_same<C, char>::value, handle>) { return false; } + bool load_raw(enable_if_t<!std::is_same<C, char>::value, handle>) { + return false; + } }; template <typename CharT, class Traits, class Allocator> -struct type_caster<std::basic_string<CharT, Traits, Allocator>, enable_if_t<is_std_char_type<CharT>::value>> +struct type_caster<std::basic_string<CharT, Traits, Allocator>, + enable_if_t<is_std_char_type<CharT>::value>> : string_caster<std::basic_string<CharT, Traits, Allocator>> {}; #ifdef PYBIND11_HAS_STRING_VIEW template <typename CharT, class Traits> -struct type_caster<std::basic_string_view<CharT, Traits>, enable_if_t<is_std_char_type<CharT>::value>> +struct type_caster<std::basic_string_view<CharT, Traits>, + enable_if_t<is_std_char_type<CharT>::value>> : string_caster<std::basic_string_view<CharT, Traits>, true> {}; #endif // Type caster for C-style strings. We basically use a std::string type caster, but also add the // ability to use None as a nullptr char* (which the string caster doesn't allow). -template <typename CharT> struct type_caster<CharT, enable_if_t<is_std_char_type<CharT>::value>> { +template <typename CharT> +struct type_caster<CharT, enable_if_t<is_std_char_type<CharT>::value>> { using StringType = std::basic_string<CharT>; - using StringCaster = type_caster<StringType>; + using StringCaster = make_caster<StringType>; StringCaster str_caster; bool none = false; CharT one_char = 0; + public: bool load(handle src, bool convert) { - if (!src) return false; + if (!src) { + return false; + } if (src.is_none()) { // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; + if (!convert) { + return false; + } none = true; return true; } @@ -1358,45 +540,58 @@ public: } static handle cast(const CharT *src, return_value_policy policy, handle parent) { - if (src == nullptr) return pybind11::none().inc_ref(); + if (src == nullptr) { + return pybind11::none().release(); + } return StringCaster::cast(StringType(src), policy, parent); } static handle cast(CharT src, return_value_policy policy, handle parent) { if (std::is_same<char, CharT>::value) { handle s = PyUnicode_DecodeLatin1((const char *) &src, 1, nullptr); - if (!s) throw error_already_set(); + if (!s) { + throw error_already_set(); + } return s; } return StringCaster::cast(StringType(1, src), policy, parent); } - operator CharT*() { return none ? nullptr : const_cast<CharT *>(static_cast<StringType &>(str_caster).c_str()); } - operator CharT&() { - if (none) + explicit operator CharT *() { + return none ? nullptr : const_cast<CharT *>(static_cast<StringType &>(str_caster).c_str()); + } + explicit operator CharT &() { + if (none) { throw value_error("Cannot convert None to a character"); + } auto &value = static_cast<StringType &>(str_caster); size_t str_len = value.size(); - if (str_len == 0) + if (str_len == 0) { throw value_error("Cannot convert empty string to a character"); + } // If we're in UTF-8 mode, we have two possible failures: one for a unicode character that - // is too high, and one for multiple unicode characters (caught later), so we need to figure - // out how long the first encoded character is in bytes to distinguish between these two - // errors. We also allow want to allow unicode characters U+0080 through U+00FF, as those - // can fit into a single char value. + // is too high, and one for multiple unicode characters (caught later), so we need to + // figure out how long the first encoded character is in bytes to distinguish between these + // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as + // those can fit into a single char value. if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { auto v0 = static_cast<unsigned char>(value[0]); - size_t char0_bytes = !(v0 & 0x80) ? 1 : // low bits only: 0-127 - (v0 & 0xE0) == 0xC0 ? 2 : // 0b110xxxxx - start of 2-byte sequence - (v0 & 0xF0) == 0xE0 ? 3 : // 0b1110xxxx - start of 3-byte sequence - 4; // 0b11110xxx - start of 4-byte sequence + // low bits only: 0-127 + // 0b110xxxxx - start of 2-byte sequence + // 0b1110xxxx - start of 3-byte sequence + // 0b11110xxx - start of 4-byte sequence + size_t char0_bytes = (v0 & 0x80) == 0 ? 1 + : (v0 & 0xE0) == 0xC0 ? 2 + : (v0 & 0xF0) == 0xE0 ? 3 + : 4; if (char0_bytes == str_len) { // If we have a 128-255 value, we can decode it into a single char: if (char0_bytes == 2 && (v0 & 0xFC) == 0xC0) { // 0x110000xx 0x10xxxxxx - one_char = static_cast<CharT>(((v0 & 3) << 6) + (static_cast<unsigned char>(value[1]) & 0x3F)); + one_char = static_cast<CharT>(((v0 & 3) << 6) + + (static_cast<unsigned char>(value[1]) & 0x3F)); return one_char; } // Otherwise we have a single character, but it's > U+00FF @@ -1409,34 +604,40 @@ public: // string was too long" error). else if (StringCaster::UTF_N == 16 && str_len == 2) { one_char = static_cast<CharT>(value[0]); - if (one_char >= 0xD800 && one_char < 0xE000) + if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); + } } - if (str_len != 1) + if (str_len != 1) { throw value_error("Expected a character, but multi-character string found"); + } one_char = value[0]; return one_char; } - static constexpr auto name = _(PYBIND11_STRING_NAME); - template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>; + static constexpr auto name = const_name(PYBIND11_STRING_NAME); + template <typename _T> + using cast_op_type = pybind11::detail::cast_op_type<_T>; }; // Base implementation for std::tuple and std::pair -template <template<typename...> class Tuple, typename... Ts> class tuple_caster { +template <template <typename...> class Tuple, typename... Ts> +class tuple_caster { using type = Tuple<Ts...>; static constexpr auto size = sizeof...(Ts); using indices = make_index_sequence<size>; -public: +public: bool load(handle src, bool convert) { - if (!isinstance<sequence>(src)) + if (!isinstance<sequence>(src)) { return false; + } const auto seq = reinterpret_borrow<sequence>(src); - if (seq.size() != size) + if (seq.size() != size) { return false; + } return load_impl(seq, convert, indices{}); } @@ -1448,66 +649,83 @@ public: // copied from the PYBIND11_TYPE_CASTER macro template <typename T> static handle cast(T *src, return_value_policy policy, handle parent) { - if (!src) return none().release(); + if (!src) { + return none().release(); + } if (policy == return_value_policy::take_ownership) { - auto h = cast(std::move(*src), policy, parent); delete src; return h; - } else { - return cast(*src, policy, parent); + auto h = cast(std::move(*src), policy, parent); + delete src; + return h; } + return cast(*src, policy, parent); } - static constexpr auto name = _("Tuple[") + concat(make_caster<Ts>::name...) + _("]"); + static constexpr auto name + = const_name("Tuple[") + concat(make_caster<Ts>::name...) + const_name("]"); - template <typename T> using cast_op_type = type; + template <typename T> + using cast_op_type = type; - operator type() & { return implicit_cast(indices{}); } - operator type() && { return std::move(*this).implicit_cast(indices{}); } + explicit operator type() & { return implicit_cast(indices{}); } + explicit operator type() && { return std::move(*this).implicit_cast(indices{}); } protected: template <size_t... Is> - type implicit_cast(index_sequence<Is...>) & { return type(cast_op<Ts>(std::get<Is>(subcasters))...); } + type implicit_cast(index_sequence<Is...>) & { + return type(cast_op<Ts>(std::get<Is>(subcasters))...); + } template <size_t... Is> - type implicit_cast(index_sequence<Is...>) && { return type(cast_op<Ts>(std::move(std::get<Is>(subcasters)))...); } + type implicit_cast(index_sequence<Is...>) && { + return type(cast_op<Ts>(std::move(std::get<Is>(subcasters)))...); + } static constexpr bool load_impl(const sequence &, bool, index_sequence<>) { return true; } template <size_t... Is> bool load_impl(const sequence &seq, bool convert, index_sequence<Is...>) { #ifdef __cpp_fold_expressions - if ((... || !std::get<Is>(subcasters).load(seq[Is], convert))) + if ((... || !std::get<Is>(subcasters).load(seq[Is], convert))) { return false; + } #else - for (bool r : {std::get<Is>(subcasters).load(seq[Is], convert)...}) - if (!r) + for (bool r : {std::get<Is>(subcasters).load(seq[Is], convert)...}) { + if (!r) { return false; + } + } #endif return true; } /* Implementation: Convert a C++ tuple into a Python tuple */ template <typename T, size_t... Is> - static handle cast_impl(T &&src, return_value_policy policy, handle parent, index_sequence<Is...>) { - std::array<object, size> entries{{ - reinterpret_steal<object>(make_caster<Ts>::cast(std::get<Is>(std::forward<T>(src)), policy, parent))... - }}; - for (const auto &entry: entries) - if (!entry) + static handle + cast_impl(T &&src, return_value_policy policy, handle parent, index_sequence<Is...>) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(src, policy, parent); + PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(policy, parent); + std::array<object, size> entries{{reinterpret_steal<object>( + make_caster<Ts>::cast(std::get<Is>(std::forward<T>(src)), policy, parent))...}}; + for (const auto &entry : entries) { + if (!entry) { return handle(); + } + } tuple result(size); int counter = 0; - for (auto & entry: entries) + for (auto &entry : entries) { PyTuple_SET_ITEM(result.ptr(), counter++, entry.release().ptr()); + } return result.release(); } Tuple<make_caster<Ts>...> subcasters; }; -template <typename T1, typename T2> class type_caster<std::pair<T1, T2>> - : public tuple_caster<std::pair, T1, T2> {}; +template <typename T1, typename T2> +class type_caster<std::pair<T1, T2>> : public tuple_caster<std::pair, T1, T2> {}; -template <typename... Ts> class type_caster<std::tuple<Ts...>> - : public tuple_caster<std::tuple, Ts...> {}; +template <typename... Ts> +class type_caster<std::tuple<Ts...>> : public tuple_caster<std::tuple, Ts...> {}; /// Helper class which abstracts away certain actions. Users can provide specializations for /// custom holders, but it's only necessary if the type has a non-standard interface. @@ -1517,12 +735,16 @@ struct holder_helper { }; /// Type caster for holder types like std::shared_ptr, etc. -template <typename type, typename holder_type> +/// The SFINAE hook is provided to help work around the current lack of support +/// for smart-pointer interoperability. Please consider it an implementation +/// detail that may change in the future, as formal support for smart-pointer +/// interoperability is added into pybind11. +template <typename type, typename holder_type, typename SFINAE = void> struct copyable_holder_caster : public type_caster_base<type> { public: using base = type_caster_base<type>; static_assert(std::is_base_of<base, type_caster<type>>::value, - "Holder classes are only supported for custom types"); + "Holder classes are only supported for custom types"); using base::base; using base::cast; using base::typeinfo; @@ -1532,12 +754,12 @@ public: return base::template load_impl<copyable_holder_caster<type, holder_type>>(src, convert); } - explicit operator type*() { return this->value; } + explicit operator type *() { return this->value; } // static_cast works around compiler error with MSVC 17 and CUDA 10.2 // see issue #2180 - explicit operator type&() { return *(static_cast<type *>(this->value)); } - explicit operator holder_type*() { return std::addressof(holder); } - explicit operator holder_type&() { return holder; } + explicit operator type &() { return *(static_cast<type *>(this->value)); } + explicit operator holder_type *() { return std::addressof(holder); } + explicit operator holder_type &() { return holder; } static handle cast(const holder_type &src, return_value_policy, handle) { const auto *ptr = holder_helper<holder_type>::get(src); @@ -1547,8 +769,9 @@ public: protected: friend class type_caster_generic; void check_holder_compat() { - if (typeinfo->default_holder) + if (typeinfo->default_holder) { throw cast_error("Unable to load a custom holder type from a default-holder instance"); + } } bool load_value(value_and_holder &&v_h) { @@ -1556,20 +779,25 @@ protected: value = v_h.value_ptr(); holder = v_h.template holder<holder_type>(); return true; - } else { - throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) " -#if defined(NDEBUG) - "(compile in debug mode for type information)"); + } + throw cast_error("Unable to cast from non-held to held instance (T& to Holder<T>) " +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for " + "type information)"); #else - "of type '" + type_id<holder_type>() + "''"); + "of type '" + + type_id<holder_type>() + "''"); #endif - } } - template <typename T = holder_type, detail::enable_if_t<!std::is_constructible<T, const T &, type*>::value, int> = 0> - bool try_implicit_casts(handle, bool) { return false; } + template <typename T = holder_type, + detail::enable_if_t<!std::is_constructible<T, const T &, type *>::value, int> = 0> + bool try_implicit_casts(handle, bool) { + return false; + } - template <typename T = holder_type, detail::enable_if_t<std::is_constructible<T, const T &, type*>::value, int> = 0> + template <typename T = holder_type, + detail::enable_if_t<std::is_constructible<T, const T &, type *>::value, int> = 0> bool try_implicit_casts(handle src, bool convert) { for (auto &cast : typeinfo->implicit_casts) { copyable_holder_caster sub_caster(*cast.first); @@ -1584,18 +812,20 @@ protected: static bool try_direct_conversions(handle) { return false; } - holder_type holder; }; /// Specialize for the common std::shared_ptr, so users don't need to template <typename T> -class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> { }; +class type_caster<std::shared_ptr<T>> : public copyable_holder_caster<T, std::shared_ptr<T>> {}; -template <typename type, typename holder_type> +/// Type caster for holder types like std::unique_ptr. +/// Please consider the SFINAE hook an implementation detail, as explained +/// in the comment for the copyable_holder_caster. +template <typename type, typename holder_type, typename SFINAE = void> struct move_only_holder_caster { static_assert(std::is_base_of<type_caster_base<type>, type_caster<type>>::value, - "Holder classes are only supported for custom types"); + "Holder classes are only supported for custom types"); static handle cast(holder_type &&src, return_value_policy, handle) { auto *ptr = holder_helper<holder_type>::get(src); @@ -1606,50 +836,101 @@ struct move_only_holder_caster { template <typename type, typename deleter> class type_caster<std::unique_ptr<type, deleter>> - : public move_only_holder_caster<type, std::unique_ptr<type, deleter>> { }; + : public move_only_holder_caster<type, std::unique_ptr<type, deleter>> {}; template <typename type, typename holder_type> using type_caster_holder = conditional_t<is_copy_constructible<holder_type>::value, copyable_holder_caster<type, holder_type>, move_only_holder_caster<type, holder_type>>; -template <typename T, bool Value = false> struct always_construct_holder { static constexpr bool value = Value; }; +template <typename T, bool Value = false> +struct always_construct_holder { + static constexpr bool value = Value; +}; /// Create a specialization for custom holder types (silently ignores std::shared_ptr) -#define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ - namespace pybind11 { namespace detail { \ - template <typename type> \ - struct always_construct_holder<holder_type> : always_construct_holder<void, ##__VA_ARGS__> { }; \ - template <typename type> \ - class type_caster<holder_type, enable_if_t<!is_shared_ptr<holder_type>::value>> \ - : public type_caster_holder<type, holder_type> { }; \ - }} +#define PYBIND11_DECLARE_HOLDER_TYPE(type, holder_type, ...) \ + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ + namespace detail { \ + template <typename type> \ + struct always_construct_holder<holder_type> : always_construct_holder<void, ##__VA_ARGS__> { \ + }; \ + template <typename type> \ + class type_caster<holder_type, enable_if_t<!is_shared_ptr<holder_type>::value>> \ + : public type_caster_holder<type, holder_type> {}; \ + } \ + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) // PYBIND11_DECLARE_HOLDER_TYPE holder types: -template <typename base, typename holder> struct is_holder_type : - std::is_base_of<detail::type_caster_holder<base, holder>, detail::type_caster<holder>> {}; +template <typename base, typename holder> +struct is_holder_type + : std::is_base_of<detail::type_caster_holder<base, holder>, detail::type_caster<holder>> {}; // Specialization for always-supported unique_ptr holders: -template <typename base, typename deleter> struct is_holder_type<base, std::unique_ptr<base, deleter>> : - std::true_type {}; - -template <typename T> struct handle_type_name { static constexpr auto name = _<T>(); }; -template <> struct handle_type_name<bytes> { static constexpr auto name = _(PYBIND11_BYTES_NAME); }; -template <> struct handle_type_name<int_> { static constexpr auto name = _("int"); }; -template <> struct handle_type_name<iterable> { static constexpr auto name = _("Iterable"); }; -template <> struct handle_type_name<iterator> { static constexpr auto name = _("Iterator"); }; -template <> struct handle_type_name<none> { static constexpr auto name = _("None"); }; -template <> struct handle_type_name<args> { static constexpr auto name = _("*args"); }; -template <> struct handle_type_name<kwargs> { static constexpr auto name = _("**kwargs"); }; +template <typename base, typename deleter> +struct is_holder_type<base, std::unique_ptr<base, deleter>> : std::true_type {}; + +template <typename T> +struct handle_type_name { + static constexpr auto name = const_name<T>(); +}; +template <> +struct handle_type_name<bool_> { + static constexpr auto name = const_name("bool"); +}; +template <> +struct handle_type_name<bytes> { + static constexpr auto name = const_name(PYBIND11_BYTES_NAME); +}; +template <> +struct handle_type_name<int_> { + static constexpr auto name = const_name("int"); +}; +template <> +struct handle_type_name<iterable> { + static constexpr auto name = const_name("Iterable"); +}; +template <> +struct handle_type_name<iterator> { + static constexpr auto name = const_name("Iterator"); +}; +template <> +struct handle_type_name<float_> { + static constexpr auto name = const_name("float"); +}; +template <> +struct handle_type_name<none> { + static constexpr auto name = const_name("None"); +}; +template <> +struct handle_type_name<args> { + static constexpr auto name = const_name("*args"); +}; +template <> +struct handle_type_name<kwargs> { + static constexpr auto name = const_name("**kwargs"); +}; template <typename type> struct pyobject_caster { template <typename T = type, enable_if_t<std::is_same<T, handle>::value, int> = 0> - bool load(handle src, bool /* convert */) { value = src; return static_cast<bool>(value); } + pyobject_caster() : value() {} + // `type` may not be default constructible (e.g. frozenset, anyset). Initializing `value` + // to a nil handle is safe since it will only be accessed if `load` succeeds. template <typename T = type, enable_if_t<std::is_base_of<object, T>::value, int> = 0> + pyobject_caster() : value(reinterpret_steal<type>(handle())) {} + + template <typename T = type, enable_if_t<std::is_same<T, handle>::value, int> = 0> bool load(handle src, bool /* convert */) { - if (!isinstance<type>(src)) + value = src; + return static_cast<bool>(value); + } + + template <typename T = type, enable_if_t<std::is_base_of<object, T>::value, int> = 0> + bool load(handle src, bool /* convert */) { + if (!isinstance<type>(src)) { return false; + } value = reinterpret_borrow<type>(src); return true; } @@ -1661,7 +942,7 @@ struct pyobject_caster { }; template <typename T> -class type_caster<T, enable_if_t<is_pyobject<T>::value>> : public pyobject_caster<T> { }; +class type_caster<T, enable_if_t<is_pyobject<T>::value>> : public pyobject_caster<T> {}; // Our conditions for enabling moving are quite restrictive: // At compile time: @@ -1672,65 +953,86 @@ class type_caster<T, enable_if_t<is_pyobject<T>::value>> : public pyobject_caste // - if the type is non-copy-constructible, the object must be the sole owner of the type (i.e. it // must have ref_count() == 1)h // If any of the above are not satisfied, we fall back to copying. -template <typename T> using move_is_plain_type = satisfies_none_of<T, - std::is_void, std::is_pointer, std::is_reference, std::is_const ->; -template <typename T, typename SFINAE = void> struct move_always : std::false_type {}; -template <typename T> struct move_always<T, enable_if_t<all_of< - move_is_plain_type<T>, - negation<is_copy_constructible<T>>, - std::is_move_constructible<T>, - std::is_same<decltype(std::declval<make_caster<T>>().operator T&()), T&> ->::value>> : std::true_type {}; -template <typename T, typename SFINAE = void> struct move_if_unreferenced : std::false_type {}; -template <typename T> struct move_if_unreferenced<T, enable_if_t<all_of< - move_is_plain_type<T>, - negation<move_always<T>>, - std::is_move_constructible<T>, - std::is_same<decltype(std::declval<make_caster<T>>().operator T&()), T&> ->::value>> : std::true_type {}; -template <typename T> using move_never = none_of<move_always<T>, move_if_unreferenced<T>>; +template <typename T> +using move_is_plain_type + = satisfies_none_of<T, std::is_void, std::is_pointer, std::is_reference, std::is_const>; +template <typename T, typename SFINAE = void> +struct move_always : std::false_type {}; +template <typename T> +struct move_always< + T, + enable_if_t< + all_of<move_is_plain_type<T>, + negation<is_copy_constructible<T>>, + is_move_constructible<T>, + std::is_same<decltype(std::declval<make_caster<T>>().operator T &()), T &>>::value>> + : std::true_type {}; +template <typename T, typename SFINAE = void> +struct move_if_unreferenced : std::false_type {}; +template <typename T> +struct move_if_unreferenced< + T, + enable_if_t< + all_of<move_is_plain_type<T>, + negation<move_always<T>>, + is_move_constructible<T>, + std::is_same<decltype(std::declval<make_caster<T>>().operator T &()), T &>>::value>> + : std::true_type {}; +template <typename T> +using move_never = none_of<move_always<T>, move_if_unreferenced<T>>; // Detect whether returning a `type` from a cast on type's type_caster is going to result in a // reference or pointer to a local variable of the type_caster. Basically, only // non-reference/pointer `type`s and reference/pointers from a type_caster_generic are safe; // everything else returns a reference/pointer to a local variable. -template <typename type> using cast_is_temporary_value_reference = bool_constant< - (std::is_reference<type>::value || std::is_pointer<type>::value) && - !std::is_base_of<type_caster_generic, make_caster<type>>::value && - !std::is_same<intrinsic_t<type>, void>::value ->; +template <typename type> +using cast_is_temporary_value_reference + = bool_constant<(std::is_reference<type>::value || std::is_pointer<type>::value) + && !std::is_base_of<type_caster_generic, make_caster<type>>::value + && !std::is_same<intrinsic_t<type>, void>::value>; // When a value returned from a C++ function is being cast back to Python, we almost always want to // force `policy = move`, regardless of the return value policy the function/method was declared // with. -template <typename Return, typename SFINAE = void> struct return_value_policy_override { +template <typename Return, typename SFINAE = void> +struct return_value_policy_override { static return_value_policy policy(return_value_policy p) { return p; } }; -template <typename Return> struct return_value_policy_override<Return, - detail::enable_if_t<std::is_base_of<type_caster_generic, make_caster<Return>>::value, void>> { +template <typename Return> +struct return_value_policy_override< + Return, + detail::enable_if_t<std::is_base_of<type_caster_generic, make_caster<Return>>::value, void>> { static return_value_policy policy(return_value_policy p) { - return !std::is_lvalue_reference<Return>::value && - !std::is_pointer<Return>::value - ? return_value_policy::move : p; + return !std::is_lvalue_reference<Return>::value && !std::is_pointer<Return>::value + ? return_value_policy::move + : p; } }; // Basic python -> C++ casting; throws if casting fails -template <typename T, typename SFINAE> type_caster<T, SFINAE> &load_type(type_caster<T, SFINAE> &conv, const handle &handle) { +template <typename T, typename SFINAE> +type_caster<T, SFINAE> &load_type(type_caster<T, SFINAE> &conv, const handle &handle) { + static_assert(!detail::is_pyobject<T>::value, + "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { -#if defined(NDEBUG) - throw cast_error("Unable to cast Python instance to C++ type (compile in debug mode for details)"); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error( + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast<std::string>() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to cast Python instance of type " + - (std::string) str(type::handle_of(handle)) + " to C++ type '" + type_id<T>() + "'"); + throw cast_error("Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast<std::string>() + " to C++ type '" + + type_id<T>() + "'"); #endif } return conv; } // Wrapper around the above that also constructs and returns a type_caster -template <typename T> make_caster<T> load_type(const handle &handle) { +template <typename T> +make_caster<T> load_type(const handle &handle) { make_caster<T> conv; load_type(conv, handle); return conv; @@ -1739,48 +1041,97 @@ template <typename T> make_caster<T> load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0> +template <typename T, + detail::enable_if_t<!detail::is_pyobject<T>::value + && !detail::is_same_ignoring_cvref<T, PyObject *>::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference<T>::value, - "Unable to cast type to reference: value is local to type caster"); + "Unable to cast type to reference: value is local to type caster"); return cast_op<T>(load_type<T>(handle)); } // pytype -> pytype (calls converting constructor) template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> -T cast(const handle &handle) { return T(reinterpret_borrow<object>(handle)); } +T cast(const handle &handle) { + return T(reinterpret_borrow<object>(handle)); +} + +// Note that `cast<PyObject *>(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast<T>()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template <typename T, + typename Handle, + detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value + && detail::is_same_ignoring_cvref<Handle, handle>::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template <typename T, + typename Object, + detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value + && detail::is_same_ignoring_cvref<Object, object>::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} // C++ type -> py::object template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0> -object cast(T &&value, return_value_policy policy = return_value_policy::automatic_reference, +object cast(T &&value, + return_value_policy policy = return_value_policy::automatic_reference, handle parent = handle()) { using no_ref_T = typename std::remove_reference<T>::type; - if (policy == return_value_policy::automatic) - policy = std::is_pointer<no_ref_T>::value ? return_value_policy::take_ownership : - std::is_lvalue_reference<T>::value ? return_value_policy::copy : return_value_policy::move; - else if (policy == return_value_policy::automatic_reference) - policy = std::is_pointer<no_ref_T>::value ? return_value_policy::reference : - std::is_lvalue_reference<T>::value ? return_value_policy::copy : return_value_policy::move; - return reinterpret_steal<object>(detail::make_caster<T>::cast(std::forward<T>(value), policy, parent)); + if (policy == return_value_policy::automatic) { + policy = std::is_pointer<no_ref_T>::value ? return_value_policy::take_ownership + : std::is_lvalue_reference<T>::value ? return_value_policy::copy + : return_value_policy::move; + } else if (policy == return_value_policy::automatic_reference) { + policy = std::is_pointer<no_ref_T>::value ? return_value_policy::reference + : std::is_lvalue_reference<T>::value ? return_value_policy::copy + : return_value_policy::move; + } + return reinterpret_steal<object>( + detail::make_caster<T>::cast(std::forward<T>(value), policy, parent)); } -template <typename T> T handle::cast() const { return pybind11::cast<T>(*this); } -template <> inline void handle::cast() const { return; } +template <typename T> +T handle::cast() const { + return pybind11::cast<T>(*this); +} +template <> +inline void handle::cast() const { + return; +} template <typename T> detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) { - if (obj.ref_count() > 1) -#if defined(NDEBUG) - throw cast_error("Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (compile in debug mode for details)"); + if (obj.ref_count() > 1) { +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error( + "Unable to cast Python " + str(type::handle_of(obj)).cast<std::string>() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) + - " instance to C++ " + type_id<T>() + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast<std::string>() + " instance to C++ " + + type_id<T>() + " instance: instance has multiple references"); #endif + } // Move into a temporary and return that, because the reference may be a local value of `conv` - T ret = std::move(detail::load_type<T>(obj).operator T&()); + T ret = std::move(detail::load_type<T>(obj).operator T &()); return ret; } @@ -1789,96 +1140,163 @@ detail::enable_if_t<!detail::move_never<T>::value, T> move(object &&obj) { // object has multiple references, but trying to copy will fail to compile. // - If both movable and copyable, check ref count: if 1, move; otherwise copy // - Otherwise (not movable), copy. -template <typename T> detail::enable_if_t<detail::move_always<T>::value, T> cast(object &&object) { +template <typename T> +detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_always<T>::value, T> +cast(object &&object) { return move<T>(std::move(object)); } -template <typename T> detail::enable_if_t<detail::move_if_unreferenced<T>::value, T> cast(object &&object) { - if (object.ref_count() > 1) +template <typename T> +detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_if_unreferenced<T>::value, T> +cast(object &&object) { + if (object.ref_count() > 1) { return cast<T>(object); - else - return move<T>(std::move(object)); + } + return move<T>(std::move(object)); } -template <typename T> detail::enable_if_t<detail::move_never<T>::value, T> cast(object &&object) { +template <typename T> +detail::enable_if_t<!detail::is_pyobject<T>::value && detail::move_never<T>::value, T> +cast(object &&object) { return cast<T>(object); } -template <typename T> T object::cast() const & { return pybind11::cast<T>(*this); } -template <typename T> T object::cast() && { return pybind11::cast<T>(std::move(*this)); } -template <> inline void object::cast() const & { return; } -template <> inline void object::cast() && { return; } +// pytype rvalue -> pytype (calls converting constructor) +template <typename T> +detail::enable_if_t<detail::is_pyobject<T>::value, T> cast(object &&object) { + return T(std::move(object)); +} + +template <typename T> +T object::cast() const & { + return pybind11::cast<T>(*this); +} +template <typename T> +T object::cast() && { + return pybind11::cast<T>(std::move(*this)); +} +template <> +inline void object::cast() const & { + return; +} +template <> +inline void object::cast() && { + return; +} PYBIND11_NAMESPACE_BEGIN(detail) // Declared in pytypes.h: template <typename T, enable_if_t<!is_pyobject<T>::value, int>> -object object_or_cast(T &&o) { return pybind11::cast(std::forward<T>(o)); } +object object_or_cast(T &&o) { + return pybind11::cast(std::forward<T>(o)); +} -struct override_unused {}; // Placeholder type for the unneeded (and dead code) static variable in the PYBIND11_OVERRIDE_OVERRIDE macro -template <typename ret_type> using override_caster_t = conditional_t< - cast_is_temporary_value_reference<ret_type>::value, make_caster<ret_type>, override_unused>; +// Placeholder type for the unneeded (and dead code) static variable in the +// PYBIND11_OVERRIDE_OVERRIDE macro +struct override_unused {}; +template <typename ret_type> +using override_caster_t = conditional_t<cast_is_temporary_value_reference<ret_type>::value, + make_caster<ret_type>, + override_unused>; // Trampoline use: for reference/pointer types to value-converted values, we do a value cast, then // store the result in the given variable. For other types, this is a no-op. -template <typename T> enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_ref(object &&o, make_caster<T> &caster) { +template <typename T> +enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_ref(object &&o, + make_caster<T> &caster) { return cast_op<T>(load_type(caster, o)); } -template <typename T> enable_if_t<!cast_is_temporary_value_reference<T>::value, T> cast_ref(object &&, override_unused &) { - pybind11_fail("Internal error: cast_ref fallback invoked"); } - -// Trampoline use: Having a pybind11::cast with an invalid reference type is going to static_assert, even -// though if it's in dead code, so we provide a "trampoline" to pybind11::cast that only does anything in -// cases where pybind11::cast is valid. -template <typename T> enable_if_t<!cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&o) { - return pybind11::cast<T>(std::move(o)); } -template <typename T> enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&) { - pybind11_fail("Internal error: cast_safe fallback invoked"); } -template <> inline void cast_safe<void>(object &&) {} +template <typename T> +enable_if_t<!cast_is_temporary_value_reference<T>::value, T> cast_ref(object &&, + override_unused &) { + pybind11_fail("Internal error: cast_ref fallback invoked"); +} + +// Trampoline use: Having a pybind11::cast with an invalid reference type is going to +// static_assert, even though if it's in dead code, so we provide a "trampoline" to pybind11::cast +// that only does anything in cases where pybind11::cast is valid. +template <typename T> +enable_if_t<cast_is_temporary_value_reference<T>::value, T> cast_safe(object &&) { + pybind11_fail("Internal error: cast_safe fallback invoked"); +} +template <typename T> +enable_if_t<std::is_void<T>::value, void> cast_safe(object &&) {} +template <typename T> +enable_if_t<detail::none_of<cast_is_temporary_value_reference<T>, std::is_void<T>>::value, T> +cast_safe(object &&o) { + return pybind11::cast<T>(std::move(o)); +} PYBIND11_NAMESPACE_END(detail) +// The overloads could coexist, i.e. the #if is not strictly speaking needed, +// but it is an easy minor optimization. +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); +} +#else +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, + const std::string &type) { + return cast_error("Unable to convert call argument '" + name + "' of type '" + type + + "' to Python object"); +} +#endif + template <return_value_policy policy = return_value_policy::automatic_reference> -tuple make_tuple() { return tuple(0); } +tuple make_tuple() { + return tuple(0); +} -template <return_value_policy policy = return_value_policy::automatic_reference, - typename... Args> tuple make_tuple(Args&&... args_) { +template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args> +tuple make_tuple(Args &&...args_) { constexpr size_t size = sizeof...(Args); - std::array<object, size> args { - { reinterpret_steal<object>(detail::make_caster<Args>::cast( - std::forward<Args>(args_), policy, nullptr))... } - }; + std::array<object, size> args{{reinterpret_steal<object>( + detail::make_caster<Args>::cast(std::forward<Args>(args_), policy, nullptr))...}}; for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { -#if defined(NDEBUG) - throw cast_error("make_tuple(): unable to convert arguments to Python object (compile in debug mode for details)"); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else - std::array<std::string, size> argtypes { {type_id<Args>()...} }; - throw cast_error("make_tuple(): unable to convert argument of type '" + - argtypes[i] + "' to Python object"); + std::array<std::string, size> argtypes{{type_id<Args>()...}}; + throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); #endif } } tuple result(size); int counter = 0; - for (auto &arg_value : args) + for (auto &arg_value : args) { PyTuple_SET_ITEM(result.ptr(), counter++, arg_value.release().ptr()); + } return result; } /// \ingroup annotations /// Annotation for arguments struct arg { - /// Constructs an argument with the name of the argument; if null or omitted, this is a positional argument. - constexpr explicit arg(const char *name = nullptr) : name(name), flag_noconvert(false), flag_none(true) { } + /// Constructs an argument with the name of the argument; if null or omitted, this is a + /// positional argument. + constexpr explicit arg(const char *name = nullptr) + : name(name), flag_noconvert(false), flag_none(true) {} /// Assign a value to this argument - template <typename T> arg_v operator=(T &&value) const; + template <typename T> + arg_v operator=(T &&value) const; /// Indicate that the type should not be converted in the type caster - arg &noconvert(bool flag = true) { flag_noconvert = flag; return *this; } + arg &noconvert(bool flag = true) { + flag_noconvert = flag; + return *this; + } /// Indicates that the argument should/shouldn't allow None (e.g. for nullable pointer args) - arg &none(bool flag = true) { flag_none = flag; return *this; } + arg &none(bool flag = true) { + flag_none = flag; + return *this; + } - const char *name; ///< If non-null, this is a named kwargs argument - bool flag_noconvert : 1; ///< If set, do not allow conversion (requires a supporting type caster!) - bool flag_none : 1; ///< If set (the default), allow None to be passed to this argument + const char *name; ///< If non-null, this is a named kwargs argument + bool flag_noconvert : 1; ///< If set, do not allow conversion (requires a supporting type + ///< caster!) + bool flag_none : 1; ///< If set (the default), allow None to be passed to this argument }; /// \ingroup annotations @@ -1887,13 +1305,12 @@ struct arg_v : arg { private: template <typename T> arg_v(arg &&base, T &&x, const char *descr = nullptr) - : arg(base), - value(reinterpret_steal<object>( - detail::make_caster<T>::cast(x, return_value_policy::automatic, {}) - )), + : arg(base), value(reinterpret_steal<object>(detail::make_caster<T>::cast( + std::forward<T>(x), return_value_policy::automatic, {}))), descr(descr) -#if !defined(NDEBUG) - , type(type_id<T>()) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + , + type(type_id<T>()) #endif { // Workaround! See: @@ -1908,44 +1325,53 @@ public: /// Direct construction with name, default, and description template <typename T> arg_v(const char *name, T &&x, const char *descr = nullptr) - : arg_v(arg(name), std::forward<T>(x), descr) { } + : arg_v(arg(name), std::forward<T>(x), descr) {} /// Called internally when invoking `py::arg("a") = value` template <typename T> arg_v(const arg &base, T &&x, const char *descr = nullptr) - : arg_v(arg(base), std::forward<T>(x), descr) { } + : arg_v(arg(base), std::forward<T>(x), descr) {} /// Same as `arg::noconvert()`, but returns *this as arg_v&, not arg& - arg_v &noconvert(bool flag = true) { arg::noconvert(flag); return *this; } + arg_v &noconvert(bool flag = true) { + arg::noconvert(flag); + return *this; + } /// Same as `arg::nonone()`, but returns *this as arg_v&, not arg& - arg_v &none(bool flag = true) { arg::none(flag); return *this; } + arg_v &none(bool flag = true) { + arg::none(flag); + return *this; + } /// The default value object value; /// The (optional) description of the default value const char *descr; -#if !defined(NDEBUG) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) /// The C++ type name of the default value (only available when compiled in debug mode) std::string type; #endif }; /// \ingroup annotations -/// Annotation indicating that all following arguments are keyword-only; the is the equivalent of an -/// unnamed '*' argument (in Python 3) +/// Annotation indicating that all following arguments are keyword-only; the is the equivalent of +/// an unnamed '*' argument struct kw_only {}; /// \ingroup annotations -/// Annotation indicating that all previous arguments are positional-only; the is the equivalent of an -/// unnamed '/' argument (in Python 3.8) +/// Annotation indicating that all previous arguments are positional-only; the is the equivalent of +/// an unnamed '/' argument (in Python 3.8) struct pos_only {}; template <typename T> -arg_v arg::operator=(T &&value) const { return {std::move(*this), std::forward<T>(value)}; } +arg_v arg::operator=(T &&value) const { + return {*this, std::forward<T>(value)}; +} /// Alias for backward compatibility -- to be removed in version 2.0 -template <typename /*unused*/> using arg_t = arg_v; +template <typename /*unused*/> +using arg_t = arg_v; inline namespace literals { /** \rst @@ -1956,6 +1382,11 @@ constexpr arg operator"" _a(const char *name, size_t) { return arg(name); } PYBIND11_NAMESPACE_BEGIN(detail) +template <typename T> +using is_kw_only = std::is_same<intrinsic_t<T>, kw_only>; +template <typename T> +using is_pos_only = std::is_same<intrinsic_t<T>, pos_only>; + // forward declaration (definition in attr.h) struct function_record; @@ -1983,56 +1414,63 @@ struct function_call { handle init_self; }; - /// Helper class which loads arguments for C++ functions called from Python template <typename... Args> class argument_loader { using indices = make_index_sequence<sizeof...(Args)>; - template <typename Arg> using argument_is_args = std::is_same<intrinsic_t<Arg>, args>; - template <typename Arg> using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>; - // Get args/kwargs argument positions relative to the end of the argument list: - static constexpr auto args_pos = constexpr_first<argument_is_args, Args...>() - (int) sizeof...(Args), - kwargs_pos = constexpr_first<argument_is_kwargs, Args...>() - (int) sizeof...(Args); + template <typename Arg> + using argument_is_args = std::is_same<intrinsic_t<Arg>, args>; + template <typename Arg> + using argument_is_kwargs = std::is_same<intrinsic_t<Arg>, kwargs>; + // Get kwargs argument position, or -1 if not present: + static constexpr auto kwargs_pos = constexpr_last<argument_is_kwargs, Args...>(); - static constexpr bool args_kwargs_are_last = kwargs_pos >= - 1 && args_pos >= kwargs_pos - 1; - - static_assert(args_kwargs_are_last, "py::args/py::kwargs are only permitted as the last argument(s) of a function"); + static_assert(kwargs_pos == -1 || kwargs_pos == (int) sizeof...(Args) - 1, + "py::kwargs is only permitted as the last argument of a function"); public: - static constexpr bool has_kwargs = kwargs_pos < 0; - static constexpr bool has_args = args_pos < 0; + static constexpr bool has_kwargs = kwargs_pos != -1; + + // py::args argument position; -1 if not present. + static constexpr int args_pos = constexpr_last<argument_is_args, Args...>(); + + static_assert(args_pos == -1 || args_pos == constexpr_first<argument_is_args, Args...>(), + "py::args cannot be specified more than once"); static constexpr auto arg_names = concat(type_descr(make_caster<Args>::name)...); - bool load_args(function_call &call) { - return load_impl_sequence(call, indices{}); - } + bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } template <typename Return, typename Guard, typename Func> + // NOLINTNEXTLINE(readability-const-return-type) enable_if_t<!std::is_void<Return>::value, Return> call(Func &&f) && { - return std::move(*this).template call_impl<Return>(std::forward<Func>(f), indices{}, Guard{}); + return std::move(*this).template call_impl<remove_cv_t<Return>>( + std::forward<Func>(f), indices{}, Guard{}); } template <typename Return, typename Guard, typename Func> enable_if_t<std::is_void<Return>::value, void_type> call(Func &&f) && { - std::move(*this).template call_impl<Return>(std::forward<Func>(f), indices{}, Guard{}); + std::move(*this).template call_impl<remove_cv_t<Return>>( + std::forward<Func>(f), indices{}, Guard{}); return void_type(); } private: - static bool load_impl_sequence(function_call &, index_sequence<>) { return true; } template <size_t... Is> bool load_impl_sequence(function_call &call, index_sequence<Is...>) { #ifdef __cpp_fold_expressions - if ((... || !std::get<Is>(argcasters).load(call.args[Is], call.args_convert[Is]))) + if ((... || !std::get<Is>(argcasters).load(call.args[Is], call.args_convert[Is]))) { return false; + } #else - for (bool r : {std::get<Is>(argcasters).load(call.args[Is], call.args_convert[Is])...}) - if (!r) + for (bool r : {std::get<Is>(argcasters).load(call.args[Is], call.args_convert[Is])...}) { + if (!r) { return false; + } + } #endif return true; } @@ -2052,7 +1490,7 @@ class simple_collector { public: template <typename... Ts> explicit simple_collector(Ts &&...values) - : m_args(pybind11::make_tuple<policy>(std::forward<Ts>(values)...)) { } + : m_args(pybind11::make_tuple<policy>(std::forward<Ts>(values)...)) {} const tuple &args() const & { return m_args; } dict kwargs() const { return {}; } @@ -2062,8 +1500,9 @@ public: /// Call a Python function and pass the collected arguments object call(PyObject *ptr) const { PyObject *result = PyObject_CallObject(ptr, m_args.ptr()); - if (!result) + if (!result) { throw error_already_set(); + } return reinterpret_steal<object>(result); } @@ -2080,8 +1519,8 @@ public: // Tuples aren't (easily) resizable so a list is needed for collection, // but the actual function call strictly requires a tuple. auto args_list = list(); - int _[] = { 0, (process(args_list, std::forward<Ts>(values)), 0)... }; - ignore_unused(_); + using expander = int[]; + (void) expander{0, (process(args_list, std::forward<Ts>(values)), 0)...}; m_args = std::move(args_list); } @@ -2095,61 +1534,66 @@ public: /// Call a Python function and pass the collected arguments object call(PyObject *ptr) const { PyObject *result = PyObject_Call(ptr, m_args.ptr(), m_kwargs.ptr()); - if (!result) + if (!result) { throw error_already_set(); + } return reinterpret_steal<object>(result); } private: template <typename T> void process(list &args_list, T &&x) { - auto o = reinterpret_steal<object>(detail::make_caster<T>::cast(std::forward<T>(x), policy, {})); + auto o = reinterpret_steal<object>( + detail::make_caster<T>::cast(std::forward<T>(x), policy, {})); if (!o) { -#if defined(NDEBUG) - argument_cast_error(); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); #else - argument_cast_error(std::to_string(args_list.size()), type_id<T>()); + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), + type_id<T>()); #endif } - args_list.append(o); + args_list.append(std::move(o)); } void process(list &args_list, detail::args_proxy ap) { - for (auto a : ap) + for (auto a : ap) { args_list.append(a); + } } - void process(list &/*args_list*/, arg_v a) { - if (!a.name) -#if defined(NDEBUG) + void process(list & /*args_list*/, arg_v a) { + if (!a.name) { +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) nameless_argument_error(); #else nameless_argument_error(a.type); #endif - + } if (m_kwargs.contains(a.name)) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else multiple_values_error(a.name); #endif } if (!a.value) { -#if defined(NDEBUG) - argument_cast_error(); +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + throw cast_error_unable_to_convert_call_arg(a.name); #else - argument_cast_error(a.name, a.type); + throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif } - m_kwargs[a.name] = a.value; + m_kwargs[a.name] = std::move(a.value); } - void process(list &/*args_list*/, detail::kwargs_proxy kp) { - if (!kp) + void process(list & /*args_list*/, detail::kwargs_proxy kp) { + if (!kp) { return; + } for (auto k : reinterpret_borrow<dict>(kp)) { if (m_kwargs.contains(k.first)) { -#if defined(NDEBUG) +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) multiple_values_error(); #else multiple_values_error(str(k.first)); @@ -2160,33 +1604,26 @@ private: } [[noreturn]] static void nameless_argument_error() { - throw type_error("Got kwargs without a name; only named arguments " - "may be passed via py::arg() to a python function call. " - "(compile in debug mode for details)"); + throw type_error( + "Got kwargs without a name; only named arguments " + "may be passed via py::arg() to a python function call. " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } - [[noreturn]] static void nameless_argument_error(std::string type) { - throw type_error("Got kwargs without a name of type '" + type + "'; only named " - "arguments may be passed via py::arg() to a python function call. "); + [[noreturn]] static void nameless_argument_error(const std::string &type) { + throw type_error("Got kwargs without a name of type '" + type + + "'; only named " + "arguments may be passed via py::arg() to a python function call. "); } [[noreturn]] static void multiple_values_error() { - throw type_error("Got multiple values for keyword argument " - "(compile in debug mode for details)"); + throw type_error( + "Got multiple values for keyword argument " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } - [[noreturn]] static void multiple_values_error(std::string name) { + [[noreturn]] static void multiple_values_error(const std::string &name) { throw type_error("Got multiple values for keyword argument '" + name + "'"); } - [[noreturn]] static void argument_cast_error() { - throw cast_error("Unable to convert call argument to Python object " - "(compile in debug mode for details)"); - } - - [[noreturn]] static void argument_cast_error(std::string name, std::string type) { - throw cast_error("Unable to convert call argument '" + name - + "' of type '" + type + "' to Python object"); - } - private: tuple m_args; dict m_kwargs; @@ -2197,35 +1634,41 @@ private: // fails to compile enable_if_t<!all_of<is_positional<Args>...>::value> // (tested with ICC 2021.1 Beta 20200827). template <typename... Args> -constexpr bool args_are_all_positional() -{ - return all_of<is_positional<Args>...>::value; +constexpr bool args_are_all_positional() { + return all_of<is_positional<Args>...>::value; } /// Collect only positional arguments for a Python function call -template <return_value_policy policy, typename... Args, +template <return_value_policy policy, + typename... Args, typename = enable_if_t<args_are_all_positional<Args...>()>> simple_collector<policy> collect_arguments(Args &&...args) { return simple_collector<policy>(std::forward<Args>(args)...); } /// Collect all arguments, including keywords and unpacking (only instantiated when needed) -template <return_value_policy policy, typename... Args, +template <return_value_policy policy, + typename... Args, typename = enable_if_t<!args_are_all_positional<Args...>()>> unpacking_collector<policy> collect_arguments(Args &&...args) { // Following argument order rules for generalized unpacking according to PEP 448 - static_assert( - constexpr_last<is_positional, Args...>() < constexpr_first<is_keyword_or_ds, Args...>() - && constexpr_last<is_s_unpacking, Args...>() < constexpr_first<is_ds_unpacking, Args...>(), - "Invalid function call: positional args must precede keywords and ** unpacking; " - "* unpacking must precede ** unpacking" - ); + static_assert(constexpr_last<is_positional, Args...>() + < constexpr_first<is_keyword_or_ds, Args...>() + && constexpr_last<is_s_unpacking, Args...>() + < constexpr_first<is_ds_unpacking, Args...>(), + "Invalid function call: positional args must precede keywords and ** unpacking; " + "* unpacking must precede ** unpacking"); return unpacking_collector<policy>(std::forward<Args>(args)...); } template <typename Derived> template <return_value_policy policy, typename... Args> object object_api<Derived>::operator()(Args &&...args) const { +#ifndef NDEBUG + if (!PyGILState_Check()) { + pybind11_fail("pybind11::object_api<>::operator() PyGILState_Check() failure."); + } +#endif return detail::collect_arguments<policy>(std::forward<Args>(args)...).call(derived().ptr()); } @@ -2237,25 +1680,25 @@ object object_api<Derived>::call(Args &&...args) const { PYBIND11_NAMESPACE_END(detail) - -template<typename T> +template <typename T> handle type::handle_of() { - static_assert( - std::is_base_of<detail::type_caster_generic, detail::make_caster<T>>::value, - "py::type::of<T> only supports the case where T is a registered C++ types." - ); + static_assert(std::is_base_of<detail::type_caster_generic, detail::make_caster<T>>::value, + "py::type::of<T> only supports the case where T is a registered C++ types."); return detail::get_type_handle(typeid(T), true); } - -#define PYBIND11_MAKE_OPAQUE(...) \ - namespace pybind11 { namespace detail { \ - template<> class type_caster<__VA_ARGS__> : public type_caster_base<__VA_ARGS__> { }; \ - }} +#define PYBIND11_MAKE_OPAQUE(...) \ + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) \ + namespace detail { \ + template <> \ + class type_caster<__VA_ARGS__> : public type_caster_base<__VA_ARGS__> {}; \ + } \ + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) /// Lets you pass a type containing a `,` through a macro parameter without needing a separate -/// typedef, e.g.: `PYBIND11_OVERRIDE(PYBIND11_TYPE(ReturnType<A, B>), PYBIND11_TYPE(Parent<C, D>), f, arg)` +/// typedef, e.g.: +/// `PYBIND11_OVERRIDE(PYBIND11_TYPE(ReturnType<A, B>), PYBIND11_TYPE(Parent<C, D>), f, arg)` #define PYBIND11_TYPE(...) __VA_ARGS__ PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index c3681102..167ea0e3 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -11,62 +11,63 @@ #pragma once #include "pybind11.h" + +#include <chrono> #include <cmath> #include <ctime> -#include <chrono> #include <datetime.h> - -// Backport the PyDateTime_DELTA functions from Python3.3 if required -#ifndef PyDateTime_DELTA_GET_DAYS -#define PyDateTime_DELTA_GET_DAYS(o) (((PyDateTime_Delta*)o)->days) -#endif -#ifndef PyDateTime_DELTA_GET_SECONDS -#define PyDateTime_DELTA_GET_SECONDS(o) (((PyDateTime_Delta*)o)->seconds) -#endif -#ifndef PyDateTime_DELTA_GET_MICROSECONDS -#define PyDateTime_DELTA_GET_MICROSECONDS(o) (((PyDateTime_Delta*)o)->microseconds) -#endif +#include <mutex> PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -template <typename type> class duration_caster { +template <typename type> +class duration_caster { public: using rep = typename type::rep; using period = typename type::period; - using days = std::chrono::duration<uint_fast32_t, std::ratio<86400>>; + // signed 25 bits required by the standard. + using days = std::chrono::duration<int_least32_t, std::ratio<86400>>; bool load(handle src, bool) { using namespace std::chrono; // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { PyDateTime_IMPORT; } + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + } - if (!src) return false; + if (!src) { + return false; + } // If invoked with datetime.delta object if (PyDelta_Check(src.ptr())) { value = type(duration_cast<duration<rep, period>>( - days(PyDateTime_DELTA_GET_DAYS(src.ptr())) + days(PyDateTime_DELTA_GET_DAYS(src.ptr())) + seconds(PyDateTime_DELTA_GET_SECONDS(src.ptr())) + microseconds(PyDateTime_DELTA_GET_MICROSECONDS(src.ptr())))); return true; } // If invoked with a float we assume it is seconds and convert - else if (PyFloat_Check(src.ptr())) { - value = type(duration_cast<duration<rep, period>>(duration<double>(PyFloat_AsDouble(src.ptr())))); + if (PyFloat_Check(src.ptr())) { + value = type(duration_cast<duration<rep, period>>( + duration<double>(PyFloat_AsDouble(src.ptr())))); return true; } - else return false; + return false; } // If this is a duration just return it back - static const std::chrono::duration<rep, period>& get_duration(const std::chrono::duration<rep, period> &src) { + static const std::chrono::duration<rep, period> & + get_duration(const std::chrono::duration<rep, period> &src) { return src; } // If this is a time_point get the time_since_epoch - template <typename Clock> static std::chrono::duration<rep, period> get_duration(const std::chrono::time_point<Clock, std::chrono::duration<rep, period>> &src) { + template <typename Clock> + static std::chrono::duration<rep, period> + get_duration(const std::chrono::time_point<Clock, std::chrono::duration<rep, period>> &src) { return src.time_since_epoch(); } @@ -78,9 +79,12 @@ public: auto d = get_duration(src); // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { PyDateTime_IMPORT; } + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + } - // Declare these special duration types so the conversions happen with the correct primitive types (int) + // Declare these special duration types so the conversions happen with the correct + // primitive types (int) using dd_t = duration<int, std::ratio<86400>>; using ss_t = duration<int, std::ratio<1>>; using us_t = duration<int, std::micro>; @@ -92,79 +96,109 @@ public: return PyDelta_FromDSU(dd.count(), ss.count(), us.count()); } - PYBIND11_TYPE_CASTER(type, _("datetime.timedelta")); + PYBIND11_TYPE_CASTER(type, const_name("datetime.timedelta")); }; +inline std::tm *localtime_thread_safe(const std::time_t *time, std::tm *buf) { +#if (defined(__STDC_LIB_EXT1__) && defined(__STDC_WANT_LIB_EXT1__)) || defined(_MSC_VER) + if (localtime_s(buf, time)) + return nullptr; + return buf; +#else + static std::mutex mtx; + std::lock_guard<std::mutex> lock(mtx); + std::tm *tm_ptr = std::localtime(time); + if (tm_ptr != nullptr) { + *buf = *tm_ptr; + } + return tm_ptr; +#endif +} + // This is for casting times on the system clock into datetime.datetime instances -template <typename Duration> class type_caster<std::chrono::time_point<std::chrono::system_clock, Duration>> { +template <typename Duration> +class type_caster<std::chrono::time_point<std::chrono::system_clock, Duration>> { public: using type = std::chrono::time_point<std::chrono::system_clock, Duration>; bool load(handle src, bool) { using namespace std::chrono; // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { PyDateTime_IMPORT; } + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + } - if (!src) return false; + if (!src) { + return false; + } std::tm cal; microseconds msecs; if (PyDateTime_Check(src.ptr())) { - cal.tm_sec = PyDateTime_DATE_GET_SECOND(src.ptr()); - cal.tm_min = PyDateTime_DATE_GET_MINUTE(src.ptr()); - cal.tm_hour = PyDateTime_DATE_GET_HOUR(src.ptr()); - cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); - cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; - cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + cal.tm_sec = PyDateTime_DATE_GET_SECOND(src.ptr()); + cal.tm_min = PyDateTime_DATE_GET_MINUTE(src.ptr()); + cal.tm_hour = PyDateTime_DATE_GET_HOUR(src.ptr()); + cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); + cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; + cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; cal.tm_isdst = -1; - msecs = microseconds(PyDateTime_DATE_GET_MICROSECOND(src.ptr())); + msecs = microseconds(PyDateTime_DATE_GET_MICROSECOND(src.ptr())); } else if (PyDate_Check(src.ptr())) { - cal.tm_sec = 0; - cal.tm_min = 0; - cal.tm_hour = 0; - cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); - cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; - cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + cal.tm_sec = 0; + cal.tm_min = 0; + cal.tm_hour = 0; + cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); + cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; + cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; cal.tm_isdst = -1; - msecs = microseconds(0); + msecs = microseconds(0); } else if (PyTime_Check(src.ptr())) { - cal.tm_sec = PyDateTime_TIME_GET_SECOND(src.ptr()); - cal.tm_min = PyDateTime_TIME_GET_MINUTE(src.ptr()); - cal.tm_hour = PyDateTime_TIME_GET_HOUR(src.ptr()); - cal.tm_mday = 1; // This date (day, month, year) = (1, 0, 70) - cal.tm_mon = 0; // represents 1-Jan-1970, which is the first - cal.tm_year = 70; // earliest available date for Python's datetime + cal.tm_sec = PyDateTime_TIME_GET_SECOND(src.ptr()); + cal.tm_min = PyDateTime_TIME_GET_MINUTE(src.ptr()); + cal.tm_hour = PyDateTime_TIME_GET_HOUR(src.ptr()); + cal.tm_mday = 1; // This date (day, month, year) = (1, 0, 70) + cal.tm_mon = 0; // represents 1-Jan-1970, which is the first + cal.tm_year = 70; // earliest available date for Python's datetime cal.tm_isdst = -1; - msecs = microseconds(PyDateTime_TIME_GET_MICROSECOND(src.ptr())); + msecs = microseconds(PyDateTime_TIME_GET_MICROSECOND(src.ptr())); + } else { + return false; } - else return false; value = time_point_cast<Duration>(system_clock::from_time_t(std::mktime(&cal)) + msecs); return true; } - static handle cast(const std::chrono::time_point<std::chrono::system_clock, Duration> &src, return_value_policy /* policy */, handle /* parent */) { + static handle cast(const std::chrono::time_point<std::chrono::system_clock, Duration> &src, + return_value_policy /* policy */, + handle /* parent */) { using namespace std::chrono; // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { PyDateTime_IMPORT; } + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + } - // Get out microseconds, and make sure they are positive, to avoid bug in eastern hemisphere time zones - // (cfr. https://github.com/pybind/pybind11/issues/2417) + // Get out microseconds, and make sure they are positive, to avoid bug in eastern + // hemisphere time zones (cfr. https://github.com/pybind/pybind11/issues/2417) using us_t = duration<int, std::micro>; auto us = duration_cast<us_t>(src.time_since_epoch() % seconds(1)); - if (us.count() < 0) + if (us.count() < 0) { us += seconds(1); + } // Subtract microseconds BEFORE `system_clock::to_time_t`, because: - // > If std::time_t has lower precision, it is implementation-defined whether the value is rounded or truncated. - // (https://en.cppreference.com/w/cpp/chrono/system_clock/to_time_t) - std::time_t tt = system_clock::to_time_t(time_point_cast<system_clock::duration>(src - us)); - // this function uses static memory so it's best to copy it out asap just in case - // otherwise other code that is using localtime may break this (not just python code) - std::tm localtime = *std::localtime(&tt); - + // > If std::time_t has lower precision, it is implementation-defined whether the value is + // rounded or truncated. (https://en.cppreference.com/w/cpp/chrono/system_clock/to_time_t) + std::time_t tt + = system_clock::to_time_t(time_point_cast<system_clock::duration>(src - us)); + + std::tm localtime; + std::tm *localtime_ptr = localtime_thread_safe(&tt, &localtime); + if (!localtime_ptr) { + throw cast_error("Unable to represent system_clock in local time"); + } return PyDateTime_FromDateAndTime(localtime.tm_year + 1900, localtime.tm_mon + 1, localtime.tm_mday, @@ -173,19 +207,19 @@ public: localtime.tm_sec, us.count()); } - PYBIND11_TYPE_CASTER(type, _("datetime.datetime")); + PYBIND11_TYPE_CASTER(type, const_name("datetime.datetime")); }; // Other clocks that are not the system clock are not measured as datetime.datetime objects // since they are not measured on calendar time. So instead we just make them timedeltas // Or if they have passed us a time as a float we convert that -template <typename Clock, typename Duration> class type_caster<std::chrono::time_point<Clock, Duration>> -: public duration_caster<std::chrono::time_point<Clock, Duration>> { -}; +template <typename Clock, typename Duration> +class type_caster<std::chrono::time_point<Clock, Duration>> + : public duration_caster<std::chrono::time_point<Clock, Duration>> {}; -template <typename Rep, typename Period> class type_caster<std::chrono::duration<Rep, Period>> -: public duration_caster<std::chrono::duration<Rep, Period>> { -}; +template <typename Rep, typename Period> +class type_caster<std::chrono::duration<Rep, Period>> + : public duration_caster<std::chrono::duration<Rep, Period>> {}; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index f8327eb3..8a831c12 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -10,42 +10,50 @@ #pragma once #include "pybind11.h" + #include <complex> /// glibc defines I as a macro which breaks things, e.g., boost template names #ifdef I -# undef I +# undef I #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) -template <typename T> struct format_descriptor<std::complex<T>, detail::enable_if_t<std::is_floating_point<T>::value>> { +template <typename T> +struct format_descriptor<std::complex<T>, detail::enable_if_t<std::is_floating_point<T>::value>> { static constexpr const char c = format_descriptor<T>::c; - static constexpr const char value[3] = { 'Z', c, '\0' }; + static constexpr const char value[3] = {'Z', c, '\0'}; static std::string format() { return std::string(value); } }; #ifndef PYBIND11_CPP17 -template <typename T> constexpr const char format_descriptor< - std::complex<T>, detail::enable_if_t<std::is_floating_point<T>::value>>::value[3]; +template <typename T> +constexpr const char + format_descriptor<std::complex<T>, + detail::enable_if_t<std::is_floating_point<T>::value>>::value[3]; #endif PYBIND11_NAMESPACE_BEGIN(detail) -template <typename T> struct is_fmt_numeric<std::complex<T>, detail::enable_if_t<std::is_floating_point<T>::value>> { +template <typename T> +struct is_fmt_numeric<std::complex<T>, detail::enable_if_t<std::is_floating_point<T>::value>> { static constexpr bool value = true; static constexpr int index = is_fmt_numeric<T>::index + 3; }; -template <typename T> class type_caster<std::complex<T>> { +template <typename T> +class type_caster<std::complex<T>> { public: bool load(handle src, bool convert) { - if (!src) + if (!src) { return false; - if (!convert && !PyComplex_Check(src.ptr())) + } + if (!convert && !PyComplex_Check(src.ptr())) { return false; + } Py_complex result = PyComplex_AsCComplex(src.ptr()); if (result.real == -1.0 && PyErr_Occurred()) { PyErr_Clear(); @@ -55,11 +63,12 @@ public: return true; } - static handle cast(const std::complex<T> &src, return_value_policy /* policy */, handle /* parent */) { + static handle + cast(const std::complex<T> &src, return_value_policy /* policy */, handle /* parent */) { return PyComplex_FromDoubles((double) src.real(), (double) src.imag()); } - PYBIND11_TYPE_CASTER(std::complex<T>, _("complex")); + PYBIND11_TYPE_CASTER(std::complex<T>, const_name("complex")); }; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 2f414e5c..bc2b40c5 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -15,13 +15,14 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) -#if PY_VERSION_HEX >= 0x03030000 && !defined(PYPY_VERSION) -# define PYBIND11_BUILTIN_QUALNAME -# define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) +#if !defined(PYPY_VERSION) +# define PYBIND11_BUILTIN_QUALNAME +# define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) #else -// In pre-3.3 Python, we still set __qualname__ so that we can produce reliable function type -// signatures; in 3.3+ this macro expands to nothing: -# define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) setattr((PyObject *) obj, "__qualname__", nameobj) +// In PyPy, we still set __qualname__ so that we can produce reliable function type +// signatures; in CPython this macro expands to nothing: +# define PYBIND11_SET_OLDPY_QUALNAME(obj, nameobj) \ + setattr((PyObject *) obj, "__qualname__", nameobj) #endif inline std::string get_fully_qualified_tp_name(PyTypeObject *type) { @@ -54,6 +55,9 @@ extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObjec return PyProperty_Type.tp_descr_set(self, cls, value); } +// Forward declaration to use in `make_static_property_type()` +inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type); + /** A `static_property` is the same as a `property` but the `__get__()` and `__set__()` methods are modified to always use the object type instead of a concrete instance. Return value: New reference. */ @@ -65,24 +69,33 @@ inline PyTypeObject *make_static_property_type() { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); - if (!heap_type) + auto *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) { pybind11_fail("make_static_property_type(): error allocating type!"); + } heap_type->ht_name = name_obj.inc_ref().ptr(); -#ifdef PYBIND11_BUILTIN_QUALNAME +# ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = name_obj.inc_ref().ptr(); -#endif +# endif - auto type = &heap_type->ht_type; + auto *type = &heap_type->ht_type; type->tp_name = name; type->tp_base = type_incref(&PyProperty_Type); type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; type->tp_descr_get = pybind11_static_get; type->tp_descr_set = pybind11_static_set; - if (PyType_Ready(type) < 0) + if (PyType_Ready(type) < 0) { pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); + } + +# if PY_VERSION_HEX >= 0x030C0000 + // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE. + // Since Python-3.12 property-derived types are required to + // have dynamic attributes (to set `__doc__`) + enable_dynamic_attributes(heap_type); +# endif setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); @@ -98,15 +111,17 @@ inline PyTypeObject *make_static_property_type() { inline PyTypeObject *make_static_property_type() { auto d = dict(); PyObject *result = PyRun_String(R"(\ - class pybind11_static_property(property): - def __get__(self, obj, cls): - return property.__get__(self, cls, cls) - - def __set__(self, obj, value): - cls = obj if isinstance(obj, type) else type(obj) - property.__set__(self, cls, value) - )", Py_file_input, d.ptr(), d.ptr() - ); +class pybind11_static_property(property): + def __get__(self, obj, cls): + return property.__get__(self, cls, cls) + + def __set__(self, obj, value): + cls = obj if isinstance(obj, type) else type(obj) + property.__set__(self, cls, value) +)", + Py_file_input, + d.ptr(), + d.ptr()); if (result == nullptr) throw error_already_set(); Py_DECREF(result); @@ -119,7 +134,7 @@ inline PyTypeObject *make_static_property_type() { By default, Python replaces the `static_property` itself, but for wrapped C++ types we need to call `static_property.__set__()` in order to propagate the new value to the underlying C++ data structure. */ -extern "C" inline int pybind11_meta_setattro(PyObject* obj, PyObject* name, PyObject* value) { +extern "C" inline int pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { // Use `_PyType_Lookup()` instead of `PyObject_GetAttr()` in order to get the raw // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name); @@ -128,9 +143,10 @@ extern "C" inline int pybind11_meta_setattro(PyObject* obj, PyObject* name, PyOb // 1. `Type.static_prop = value` --> descr_set: `Type.static_prop.__set__(value)` // 2. `Type.static_prop = other_static_prop` --> setattro: replace existing `static_prop` // 3. `Type.regular_attribute = value` --> setattro: regular attribute assignment - const auto static_prop = (PyObject *) get_internals().static_property_type; - const auto call_descr_set = descr && value && PyObject_IsInstance(descr, static_prop) - && !PyObject_IsInstance(value, static_prop); + auto *const static_prop = (PyObject *) get_internals().static_property_type; + const auto call_descr_set = (descr != nullptr) && (value != nullptr) + && (PyObject_IsInstance(descr, static_prop) != 0) + && (PyObject_IsInstance(value, static_prop) == 0); if (call_descr_set) { // Call `static_property.__set__()` instead of replacing the `static_property`. #if !defined(PYPY_VERSION) @@ -149,7 +165,6 @@ extern "C" inline int pybind11_meta_setattro(PyObject* obj, PyObject* name, PyOb } } -#if PY_MAJOR_VERSION >= 3 /** * Python 3's PyInstanceMethod_Type hides itself via its tp_descr_get, which prevents aliasing * methods via cls.attr("m2") = cls.attr("m1"): instead the tp_descr_get returns a plain function, @@ -162,11 +177,8 @@ extern "C" inline PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name Py_INCREF(descr); return descr; } - else { - return PyType_Type.tp_getattro(obj, name); - } + return PyType_Type.tp_getattro(obj, name); } -#endif /// metaclass `__call__` function that is used to create all pybind11 objects. extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, PyObject *kwargs) { @@ -178,12 +190,13 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P } // This must be a pybind11 instance - auto instance = reinterpret_cast<detail::instance *>(self); + auto *instance = reinterpret_cast<detail::instance *>(self); // Ensure that the base __init__ function(s) were called for (const auto &vh : values_and_holders(instance)) { if (!vh.holder_constructed()) { - PyErr_Format(PyExc_TypeError, "%.200s.__init__() must be called when overriding __init__", + PyErr_Format(PyExc_TypeError, + "%.200s.__init__() must be called when overriding __init__", get_fully_qualified_tp_name(vh.type->type).c_str()); Py_DECREF(self); return nullptr; @@ -202,27 +215,28 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { // 1) be found in internals.registered_types_py // 2) have exactly one associated `detail::type_info` auto found_type = internals.registered_types_py.find(type); - if (found_type != internals.registered_types_py.end() && - found_type->second.size() == 1 && - found_type->second[0]->type == type) { + if (found_type != internals.registered_types_py.end() && found_type->second.size() == 1 + && found_type->second[0]->type == type) { auto *tinfo = found_type->second[0]; auto tindex = std::type_index(*tinfo->cpptype); internals.direct_conversions.erase(tindex); - if (tinfo->module_local) - registered_local_types_cpp().erase(tindex); - else + if (tinfo->module_local) { + get_local_internals().registered_types_cpp.erase(tindex); + } else { internals.registered_types_cpp.erase(tindex); + } internals.registered_types_py.erase(tinfo->type); // Actually just `std::erase_if`, but that's only available in C++20 auto &cache = internals.inactive_override_cache; - for (auto it = cache.begin(), last = cache.end(); it != last; ) { - if (it->first == (PyObject *) tinfo->type) + for (auto it = cache.begin(), last = cache.end(); it != last;) { + if (it->first == (PyObject *) tinfo->type) { it = cache.erase(it); - else + } else { ++it; + } } delete tinfo; @@ -234,7 +248,7 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) { /** This metaclass is assigned by default to all pybind11 types and is required in order for static properties to function correctly. Users may override this using `py::metaclass`. Return value: New reference. */ -inline PyTypeObject* make_default_metaclass() { +inline PyTypeObject *make_default_metaclass() { constexpr auto *name = "pybind11_type"; auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name)); @@ -242,16 +256,17 @@ inline PyTypeObject* make_default_metaclass() { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); - if (!heap_type) + auto *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0); + if (!heap_type) { pybind11_fail("make_default_metaclass(): error allocating metaclass!"); + } heap_type->ht_name = name_obj.inc_ref().ptr(); #ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = name_obj.inc_ref().ptr(); #endif - auto type = &heap_type->ht_type; + auto *type = &heap_type->ht_type; type->tp_name = name; type->tp_base = type_incref(&PyType_Type); type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE; @@ -259,14 +274,13 @@ inline PyTypeObject* make_default_metaclass() { type->tp_call = pybind11_meta_call; type->tp_setattro = pybind11_meta_setattro; -#if PY_MAJOR_VERSION >= 3 type->tp_getattro = pybind11_meta_getattro; -#endif type->tp_dealloc = pybind11_meta_dealloc; - if (PyType_Ready(type) < 0) + if (PyType_Ready(type) < 0) { pybind11_fail("make_default_metaclass(): failure in PyType_Ready()!"); + } setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); @@ -276,16 +290,20 @@ inline PyTypeObject* make_default_metaclass() { /// For multiple inheritance types we need to recursively register/deregister base pointers for any /// base classes with pointers that are difference from the instance value pointer so that we can -/// correctly recognize an offset base class pointer. This calls a function with any offset base ptrs. -inline void traverse_offset_bases(void *valueptr, const detail::type_info *tinfo, instance *self, - bool (*f)(void * /*parentptr*/, instance * /*self*/)) { +/// correctly recognize an offset base class pointer. This calls a function with any offset base +/// ptrs. +inline void traverse_offset_bases(void *valueptr, + const detail::type_info *tinfo, + instance *self, + bool (*f)(void * /*parentptr*/, instance * /*self*/)) { for (handle h : reinterpret_borrow<tuple>(tinfo->type->tp_bases)) { - if (auto parent_tinfo = get_type_info((PyTypeObject *) h.ptr())) { + if (auto *parent_tinfo = get_type_info((PyTypeObject *) h.ptr())) { for (auto &c : parent_tinfo->implicit_casts) { if (c.first == tinfo->cpptype) { auto *parentptr = c.second(valueptr); - if (parentptr != valueptr) + if (parentptr != valueptr) { f(parentptr, self); + } traverse_offset_bases(parentptr, parent_tinfo, self, f); break; } @@ -312,31 +330,33 @@ inline bool deregister_instance_impl(void *ptr, instance *self) { inline void register_instance(instance *self, void *valptr, const type_info *tinfo) { register_instance_impl(valptr, self); - if (!tinfo->simple_ancestors) + if (!tinfo->simple_ancestors) { traverse_offset_bases(valptr, tinfo, self, register_instance_impl); + } } inline bool deregister_instance(instance *self, void *valptr, const type_info *tinfo) { bool ret = deregister_instance_impl(valptr, self); - if (!tinfo->simple_ancestors) + if (!tinfo->simple_ancestors) { traverse_offset_bases(valptr, tinfo, self, deregister_instance_impl); + } return ret; } -/// Instance creation function for all pybind11 types. It allocates the internal instance layout for -/// holding C++ objects and holders. Allocation is done lazily (the first time the instance is cast -/// to a reference or pointer), and initialization is done by an `__init__` function. +/// Instance creation function for all pybind11 types. It allocates the internal instance layout +/// for holding C++ objects and holders. Allocation is done lazily (the first time the instance is +/// cast to a reference or pointer), and initialization is done by an `__init__` function. inline PyObject *make_new_instance(PyTypeObject *type) { #if defined(PYPY_VERSION) - // PyPy gets tp_basicsize wrong (issue 2482) under multiple inheritance when the first inherited - // object is a a plain Python type (i.e. not derived from an extension type). Fix it. + // PyPy gets tp_basicsize wrong (issue 2482) under multiple inheritance when the first + // inherited object is a plain Python type (i.e. not derived from an extension type). Fix it. ssize_t instance_size = static_cast<ssize_t>(sizeof(instance)); if (type->tp_basicsize < instance_size) { type->tp_basicsize = instance_size; } #endif PyObject *self = type->tp_alloc(type, 0); - auto inst = reinterpret_cast<instance *>(self); + auto *inst = reinterpret_cast<instance *>(self); // Allocate the value/holder internals: inst->allocate_layout(); @@ -361,14 +381,14 @@ extern "C" inline int pybind11_object_init(PyObject *self, PyObject *, PyObject inline void add_patient(PyObject *nurse, PyObject *patient) { auto &internals = get_internals(); - auto instance = reinterpret_cast<detail::instance *>(nurse); + auto *instance = reinterpret_cast<detail::instance *>(nurse); instance->has_patients = true; Py_INCREF(patient); internals.patients[nurse].push_back(patient); } inline void clear_patients(PyObject *self) { - auto instance = reinterpret_cast<detail::instance *>(self); + auto *instance = reinterpret_cast<detail::instance *>(self); auto &internals = get_internals(); auto pos = internals.patients.find(self); assert(pos != internals.patients.end()); @@ -378,14 +398,15 @@ inline void clear_patients(PyObject *self) { auto patients = std::move(pos->second); internals.patients.erase(pos); instance->has_patients = false; - for (PyObject *&patient : patients) + for (PyObject *&patient : patients) { Py_CLEAR(patient); + } } /// Clears all internal data from the instance and removes it from registered instances in /// preparation for deallocation. inline void clear_instance(PyObject *self) { - auto instance = reinterpret_cast<detail::instance *>(self); + auto *instance = reinterpret_cast<detail::instance *>(self); // Deallocate any values/holders, if present: for (auto &v_h : values_and_holders(instance)) { @@ -393,33 +414,48 @@ inline void clear_instance(PyObject *self) { // We have to deregister before we call dealloc because, for virtual MI types, we still // need to be able to get the parent pointers. - if (v_h.instance_registered() && !deregister_instance(instance, v_h.value_ptr(), v_h.type)) - pybind11_fail("pybind11_object_dealloc(): Tried to deallocate unregistered instance!"); + if (v_h.instance_registered() + && !deregister_instance(instance, v_h.value_ptr(), v_h.type)) { + pybind11_fail( + "pybind11_object_dealloc(): Tried to deallocate unregistered instance!"); + } - if (instance->owned || v_h.holder_constructed()) + if (instance->owned || v_h.holder_constructed()) { v_h.type->dealloc(v_h); + } } } // Deallocate the value/holder layout internals: instance->deallocate_layout(); - if (instance->weakrefs) + if (instance->weakrefs) { PyObject_ClearWeakRefs(self); + } PyObject **dict_ptr = _PyObject_GetDictPtr(self); - if (dict_ptr) + if (dict_ptr) { Py_CLEAR(*dict_ptr); + } - if (instance->has_patients) + if (instance->has_patients) { clear_patients(self); + } } /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 @@ -437,6 +473,8 @@ extern "C" inline void pybind11_object_dealloc(PyObject *self) { #endif } +std::string error_string(); + /** Create the type which can be used as a common base for all classes. This is needed in order to satisfy Python's requirements for multiple inheritance. Return value: New reference. */ @@ -448,16 +486,17 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); - if (!heap_type) + auto *heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + if (!heap_type) { pybind11_fail("make_object_base_type(): error allocating type!"); + } heap_type->ht_name = name_obj.inc_ref().ptr(); #ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = name_obj.inc_ref().ptr(); #endif - auto type = &heap_type->ht_type; + auto *type = &heap_type->ht_type; type->tp_name = name; type->tp_base = type_incref(&PyBaseObject_Type); type->tp_basicsize = static_cast<ssize_t>(sizeof(instance)); @@ -470,8 +509,9 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { /* Support weak references (needed for the keep_alive feature) */ type->tp_weaklistoffset = offsetof(instance, weakrefs); - if (PyType_Ready(type) < 0) - pybind11_fail("PyType_Ready failed in make_object_base_type():" + error_string()); + if (PyType_Ready(type) < 0) { + pybind11_fail("PyType_Ready failed in make_object_base_type(): " + error_string()); + } setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); @@ -480,33 +520,14 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { return (PyObject *) heap_type; } -/// dynamic_attr: Support for `d = instance.__dict__`. -extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) { - PyObject *&dict = *_PyObject_GetDictPtr(self); - if (!dict) - dict = PyDict_New(); - Py_XINCREF(dict); - return dict; -} - -/// dynamic_attr: Support for `instance.__dict__ = dict()`. -extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) { - if (!PyDict_Check(new_dict)) { - PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'", - get_fully_qualified_tp_name(Py_TYPE(new_dict)).c_str()); - return -1; - } - PyObject *&dict = *_PyObject_GetDictPtr(self); - Py_INCREF(new_dict); - Py_CLEAR(dict); - dict = new_dict; - return 0; -} - /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { PyObject *&dict = *_PyObject_GetDictPtr(self); Py_VISIT(dict); +// https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse +#if PY_VERSION_HEX >= 0x03090000 + Py_VISIT(Py_TYPE(self)); +#endif return 0; } @@ -519,17 +540,28 @@ extern "C" inline int pybind11_clear(PyObject *self) { /// Give instances of this type a `__dict__` and opt into garbage collection. inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { - auto type = &heap_type->ht_type; + auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; - type->tp_dictoffset = type->tp_basicsize; // place dict at the end - type->tp_basicsize += (ssize_t)sizeof(PyObject *); // and allocate enough space for it +#if PY_VERSION_HEX < 0x030B0000 + type->tp_dictoffset = type->tp_basicsize; // place dict at the end + type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it +#else + type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; +#endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; - static PyGetSetDef getset[] = { - {const_cast<char*>("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr} - }; + static PyGetSetDef getset[] = {{ +#if PY_VERSION_HEX < 0x03070000 + const_cast<char *>("__dict__"), +#else + "__dict__", +#endif + PyObject_GenericGetDict, + PyObject_GenericSetDict, + nullptr, + nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; type->tp_getset = getset; } @@ -539,12 +571,14 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla type_info *tinfo = nullptr; for (auto type : reinterpret_borrow<tuple>(Py_TYPE(obj)->tp_mro)) { tinfo = get_type_info((PyTypeObject *) type.ptr()); - if (tinfo && tinfo->get_buffer) + if (tinfo && tinfo->get_buffer) { break; + } } if (view == nullptr || !tinfo || !tinfo->get_buffer) { - if (view) + if (view) { view->obj = nullptr; + } PyErr_SetString(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); return -1; } @@ -562,15 +596,17 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla view->buf = info->ptr; view->itemsize = info->itemsize; view->len = view->itemsize; - for (auto s : info->shape) + for (auto s : info->shape) { view->len *= s; - view->readonly = info->readonly; - if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) + } + view->readonly = static_cast<int>(info->readonly); + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { view->format = const_cast<char *>(info->format.c_str()); + } if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { view->ndim = (int) info->ndim; - view->strides = &info->strides[0]; - view->shape = &info->shape[0]; + view->strides = info->strides.data(); + view->shape = info->shape.data(); } Py_INCREF(view->obj); return 0; @@ -584,9 +620,6 @@ extern "C" inline void pybind11_releasebuffer(PyObject *, Py_buffer *view) { /// Give this type a buffer interface. inline void enable_buffer_protocol(PyHeapTypeObject *heap_type) { heap_type->ht_type.tp_as_buffer = &heap_type->as_buffer; -#if PY_MAJOR_VERSION < 3 - heap_type->ht_type.tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER; -#endif heap_type->as_buffer.bf_getbuffer = pybind11_getbuffer; heap_type->as_buffer.bf_releasebuffer = pybind11_releasebuffer; @@ -594,70 +627,68 @@ inline void enable_buffer_protocol(PyHeapTypeObject *heap_type) { /** Create a brand new Python type according to the `type_record` specification. Return value: New reference. */ -inline PyObject* make_new_python_type(const type_record &rec) { +inline PyObject *make_new_python_type(const type_record &rec) { auto name = reinterpret_steal<object>(PYBIND11_FROM_STRING(rec.name)); auto qualname = name; if (rec.scope && !PyModule_Check(rec.scope.ptr()) && hasattr(rec.scope, "__qualname__")) { -#if PY_MAJOR_VERSION >= 3 qualname = reinterpret_steal<object>( PyUnicode_FromFormat("%U.%U", rec.scope.attr("__qualname__").ptr(), name.ptr())); -#else - qualname = str(rec.scope.attr("__qualname__").cast<std::string>() + "." + rec.name); -#endif } object module_; if (rec.scope) { - if (hasattr(rec.scope, "__module__")) + if (hasattr(rec.scope, "__module__")) { module_ = rec.scope.attr("__module__"); - else if (hasattr(rec.scope, "__name__")) + } else if (hasattr(rec.scope, "__name__")) { module_ = rec.scope.attr("__name__"); + } } - auto full_name = c_str( + const auto *full_name = c_str( #if !defined(PYPY_VERSION) module_ ? str(module_).cast<std::string>() + "." + rec.name : #endif - rec.name); + rec.name); char *tp_doc = nullptr; if (rec.doc && options::show_user_defined_docstrings()) { /* Allocate memory for docstring (using PyObject_MALLOC, since Python will free this later on) */ - size_t size = strlen(rec.doc) + 1; + size_t size = std::strlen(rec.doc) + 1; tp_doc = (char *) PyObject_MALLOC(size); - memcpy((void *) tp_doc, rec.doc, size); + std::memcpy((void *) tp_doc, rec.doc, size); } auto &internals = get_internals(); auto bases = tuple(rec.bases); - auto base = (bases.empty()) ? internals.instance_base - : bases[0].ptr(); + auto *base = (bases.empty()) ? internals.instance_base : bases[0].ptr(); /* Danger zone: from now (and until PyType_Ready), make sure to issue no Python C API calls which could potentially invoke the garbage collector (the GC will call type_traverse(), which will in turn find the newly constructed type in an invalid state) */ - auto metaclass = rec.metaclass.ptr() ? (PyTypeObject *) rec.metaclass.ptr() - : internals.default_metaclass; + auto *metaclass + = rec.metaclass.ptr() ? (PyTypeObject *) rec.metaclass.ptr() : internals.default_metaclass; - auto heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); - if (!heap_type) + auto *heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0); + if (!heap_type) { pybind11_fail(std::string(rec.name) + ": Unable to create type object!"); + } heap_type->ht_name = name.release().ptr(); #ifdef PYBIND11_BUILTIN_QUALNAME heap_type->ht_qualname = qualname.inc_ref().ptr(); #endif - auto type = &heap_type->ht_type; + auto *type = &heap_type->ht_type; type->tp_name = full_name; type->tp_doc = tp_doc; - type->tp_base = type_incref((PyTypeObject *)base); + type->tp_base = type_incref((PyTypeObject *) base); type->tp_basicsize = static_cast<ssize_t>(sizeof(instance)); - if (!bases.empty()) + if (!bases.empty()) { type->tp_bases = bases.release().ptr(); + } /* Don't inherit base __init__ */ type->tp_init = pybind11_object_init; @@ -666,38 +697,42 @@ inline PyObject* make_new_python_type(const type_record &rec) { type->tp_as_number = &heap_type->as_number; type->tp_as_sequence = &heap_type->as_sequence; type->tp_as_mapping = &heap_type->as_mapping; -#if PY_VERSION_HEX >= 0x03050000 type->tp_as_async = &heap_type->as_async; -#endif /* Flags */ type->tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE; -#if PY_MAJOR_VERSION < 3 - type->tp_flags |= Py_TPFLAGS_CHECKTYPES; -#endif - if (!rec.is_final) + if (!rec.is_final) { type->tp_flags |= Py_TPFLAGS_BASETYPE; + } - if (rec.dynamic_attr) + if (rec.dynamic_attr) { enable_dynamic_attributes(heap_type); + } - if (rec.buffer_protocol) + if (rec.buffer_protocol) { enable_buffer_protocol(heap_type); + } - if (PyType_Ready(type) < 0) - pybind11_fail(std::string(rec.name) + ": PyType_Ready failed (" + error_string() + ")!"); + if (rec.custom_type_setup_callback) { + rec.custom_type_setup_callback(heap_type); + } + + if (PyType_Ready(type) < 0) { + pybind11_fail(std::string(rec.name) + ": PyType_Ready failed: " + error_string()); + } - assert(rec.dynamic_attr ? PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) - : !PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); + assert(!rec.dynamic_attr || PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)); /* Register type with the parent scope */ - if (rec.scope) + if (rec.scope) { setattr(rec.scope, rec.name, (PyObject *) type); - else + } else { Py_INCREF(type); // Keep it alive forever (reference leak) + } - if (module_) // Needed by pydoc + if (module_) { // Needed by pydoc setattr((PyObject *) type, "__module__", module_); + } PYBIND11_SET_OLDPY_QUALNAME(type, qualname); diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index de495e4f..27c426db 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -10,118 +10,264 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 6 -#define PYBIND11_VERSION_PATCH 2 +#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_PATCH 0 + +// Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html +// Additional convention: 0xD = dev +#define PYBIND11_VERSION_HEX 0x020B0000 + +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif + +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH -#define PYBIND11_NAMESPACE_BEGIN(name) namespace name { -#define PYBIND11_NAMESPACE_END(name) } +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } // Robust support for some features and loading modules compiled against different pybind versions -// requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute on -// the main `pybind11` namespace. +// requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute +// on the main `pybind11` namespace. #if !defined(PYBIND11_NAMESPACE) -# ifdef __GNUG__ -# define PYBIND11_NAMESPACE pybind11 __attribute__((visibility("hidden"))) -# else -# define PYBIND11_NAMESPACE pybind11 -# endif +# ifdef __GNUG__ +# define PYBIND11_NAMESPACE pybind11 __attribute__((visibility("hidden"))) +# else +# define PYBIND11_NAMESPACE pybind11 +# endif #endif #if !(defined(_MSC_VER) && __cplusplus == 199711L) -# if __cplusplus >= 201402L -# define PYBIND11_CPP14 -# if __cplusplus >= 201703L -# define PYBIND11_CPP17 +# if __cplusplus >= 201402L +# define PYBIND11_CPP14 +# if __cplusplus >= 201703L +# define PYBIND11_CPP17 +# if __cplusplus >= 202002L +# define PYBIND11_CPP20 +// Please update tests/pybind11_tests.cpp `cpp_std()` when adding a macro here. +# endif +# endif # endif -# endif #elif defined(_MSC_VER) && __cplusplus == 199711L -// MSVC sets _MSVC_LANG rather than __cplusplus (supposedly until the standard is fully implemented) -// Unless you use the /Zc:__cplusplus flag on Visual Studio 2017 15.7 Preview 3 or newer -# if _MSVC_LANG >= 201402L -# define PYBIND11_CPP14 -# if _MSVC_LANG > 201402L && _MSC_VER >= 1910 -# define PYBIND11_CPP17 +// MSVC sets _MSVC_LANG rather than __cplusplus (supposedly until the standard is fully +// implemented). Unless you use the /Zc:__cplusplus flag on Visual Studio 2017 15.7 Preview 3 +// or newer. +# if _MSVC_LANG >= 201402L +# define PYBIND11_CPP14 +# if _MSVC_LANG > 201402L +# define PYBIND11_CPP17 +# if _MSVC_LANG >= 202002L +# define PYBIND11_CPP20 +# endif +# endif # endif -# endif #endif // Compiler version assertions #if defined(__INTEL_COMPILER) -# if __INTEL_COMPILER < 1800 -# error pybind11 requires Intel C++ compiler v18 or newer -# elif __INTEL_COMPILER < 1900 && defined(PYBIND11_CPP14) -# error pybind11 supports only C++11 with Intel C++ compiler v18. Use v19 or newer for C++14. -# endif +# if __INTEL_COMPILER < 1800 +# error pybind11 requires Intel C++ compiler v18 or newer +# elif __INTEL_COMPILER < 1900 && defined(PYBIND11_CPP14) +# error pybind11 supports only C++11 with Intel C++ compiler v18. Use v19 or newer for C++14. +# endif +/* The following pragma cannot be pop'ed: + https://community.intel.com/t5/Intel-C-Compiler/Inline-and-no-inline-warning/td-p/1216764 */ +# pragma warning disable 2196 // warning #2196: routine is both "inline" and "noinline" #elif defined(__clang__) && !defined(__apple_build_version__) -# if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3) -# error pybind11 requires clang 3.3 or newer -# endif +# if __clang_major__ < 3 || (__clang_major__ == 3 && __clang_minor__ < 3) +# error pybind11 requires clang 3.3 or newer +# endif #elif defined(__clang__) // Apple changes clang version macros to its Xcode version; the first Xcode release based on // (upstream) clang 3.3 was Xcode 5: -# if __clang_major__ < 5 -# error pybind11 requires Xcode/clang 5.0 or newer -# endif +# if __clang_major__ < 5 +# error pybind11 requires Xcode/clang 5.0 or newer +# endif #elif defined(__GNUG__) -# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) -# error pybind11 requires gcc 4.8 or newer -# endif +# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) +# error pybind11 requires gcc 4.8 or newer +# endif #elif defined(_MSC_VER) -// Pybind hits various compiler bugs in 2015u2 and earlier, and also makes use of some stl features -// (e.g. std::negation) added in 2015u3: -# if _MSC_FULL_VER < 190024210 -# error pybind11 requires MSVC 2015 update 3 or newer -# endif +# if _MSC_VER < 1910 +# error pybind11 2.10+ requires MSVC 2017 or newer +# endif #endif #if !defined(PYBIND11_EXPORT) -# if defined(WIN32) || defined(_WIN32) -# define PYBIND11_EXPORT __declspec(dllexport) -# else -# define PYBIND11_EXPORT __attribute__ ((visibility("default"))) -# endif +# if defined(WIN32) || defined(_WIN32) +# define PYBIND11_EXPORT __declspec(dllexport) +# else +# define PYBIND11_EXPORT __attribute__((visibility("default"))) +# endif #endif -#if defined(_MSC_VER) -# define PYBIND11_NOINLINE __declspec(noinline) +#if !defined(PYBIND11_EXPORT_EXCEPTION) +# if defined(__apple_build_version__) +# define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION +# endif +#endif + +// For CUDA, GCC7, GCC8: +// PYBIND11_NOINLINE_FORCED is incompatible with `-Wattributes -Werror`. +// When defining PYBIND11_NOINLINE_FORCED, it is best to also use `-Wno-attributes`. +// However, the measured shared-library size saving when using noinline are only +// 1.7% for CUDA, -0.2% for GCC7, and 0.0% for GCC8 (using -DCMAKE_BUILD_TYPE=MinSizeRel, +// the default under pybind11/tests). +#if !defined(PYBIND11_NOINLINE_FORCED) \ + && (defined(__CUDACC__) || (defined(__GNUC__) && (__GNUC__ == 7 || __GNUC__ == 8))) +# define PYBIND11_NOINLINE_DISABLED +#endif + +// The PYBIND11_NOINLINE macro is for function DEFINITIONS. +// In contrast, FORWARD DECLARATIONS should never use this macro: +// https://stackoverflow.com/questions/9317473/forward-declaration-of-inline-functions +#if defined(PYBIND11_NOINLINE_DISABLED) // Option for maximum portability and experimentation. +# define PYBIND11_NOINLINE inline +#elif defined(_MSC_VER) +# define PYBIND11_NOINLINE __declspec(noinline) inline #else -# define PYBIND11_NOINLINE __attribute__ ((noinline)) +# define PYBIND11_NOINLINE __attribute__((noinline)) inline #endif -#if defined(PYBIND11_CPP14) -# define PYBIND11_DEPRECATED(reason) [[deprecated(reason)]] +#if defined(__MINGW32__) +// For unknown reasons all PYBIND11_DEPRECATED member trigger a warning when declared +// whether it is used or not +# define PYBIND11_DEPRECATED(reason) +#elif defined(PYBIND11_CPP14) +# define PYBIND11_DEPRECATED(reason) [[deprecated(reason)]] #else -# define PYBIND11_DEPRECATED(reason) __attribute__((deprecated(reason))) +# define PYBIND11_DEPRECATED(reason) __attribute__((deprecated(reason))) #endif #if defined(PYBIND11_CPP17) -# define PYBIND11_MAYBE_UNUSED [[maybe_unused]] +# define PYBIND11_MAYBE_UNUSED [[maybe_unused]] #elif defined(_MSC_VER) && !defined(__clang__) -# define PYBIND11_MAYBE_UNUSED +# define PYBIND11_MAYBE_UNUSED #else -# define PYBIND11_MAYBE_UNUSED __attribute__ ((__unused__)) +# define PYBIND11_MAYBE_UNUSED __attribute__((__unused__)) #endif /* Don't let Python.h #define (v)snprintf as macro because they are implemented properly in Visual Studio since 2015. */ -#if defined(_MSC_VER) && _MSC_VER >= 1900 -# define HAVE_SNPRINTF 1 +#if defined(_MSC_VER) +# define HAVE_SNPRINTF 1 #endif /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# if (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION < 4) -# define HAVE_ROUND 1 -# endif -# pragma warning(push) -# pragma warning(disable: 4510 4610 4512 4005) -# if defined(_DEBUG) && !defined(Py_DEBUG) -# define PYBIND11_DEBUG_MARKER -# undef _DEBUG -# endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4505) +// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) +# if defined(_DEBUG) && !defined(Py_DEBUG) +// Workaround for a VS 2022 issue. +// NOTE: This workaround knowingly violates the Python.h include order requirement: +// https://docs.python.org/3/c-api/intro.html#include-files +// See https://github.com/pybind/pybind11/pull/3497 for full context. +# include <yvals.h> +# if _MSVC_STL_VERSION >= 143 +# include <crtdefs.h> +# endif +# define PYBIND11_DEBUG_MARKER +# undef _DEBUG +# endif +#endif + +// https://en.cppreference.com/w/c/chrono/localtime +#if defined(__STDC_LIB_EXT1__) && !defined(__STDC_WANT_LIB_EXT1__) +# define __STDC_WANT_LIB_EXT1__ +#endif + +#ifdef __has_include +// std::optional (but including it in c++14 mode isn't allowed) +# if defined(PYBIND11_CPP17) && __has_include(<optional>) +# define PYBIND11_HAS_OPTIONAL 1 +# endif +// std::experimental::optional (but not allowed in c++11 mode) +# if defined(PYBIND11_CPP14) && (__has_include(<experimental/optional>) && \ + !__has_include(<optional>)) +# define PYBIND11_HAS_EXP_OPTIONAL 1 +# endif +// std::variant +# if defined(PYBIND11_CPP17) && __has_include(<variant>) +# define PYBIND11_HAS_VARIANT 1 +# endif +#elif defined(_MSC_VER) && defined(PYBIND11_CPP17) +# define PYBIND11_HAS_OPTIONAL 1 +# define PYBIND11_HAS_VARIANT 1 +#endif + +#if defined(PYBIND11_CPP17) +# if defined(__has_include) +# if __has_include(<string_view>) +# define PYBIND11_HAS_STRING_VIEW +# endif +# elif defined(_MSC_VER) +# define PYBIND11_HAS_STRING_VIEW +# endif #endif #include <Python.h> +// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 +#if PY_VERSION_HEX < 0x03060000 +# error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." +#endif #include <frameobject.h> #include <pythread.h> @@ -129,41 +275,77 @@ tends to weak havok in C++ codebases that expect these to work like regular functions (potentially with several overloads) */ #if defined(isalnum) -# undef isalnum -# undef isalpha -# undef islower -# undef isspace -# undef isupper -# undef tolower -# undef toupper +# undef isalnum +# undef isalpha +# undef islower +# undef isspace +# undef isupper +# undef tolower +# undef toupper #endif #if defined(copysign) -# undef copysign +# undef copysign +#endif + +#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT #endif #if defined(_MSC_VER) -# if defined(PYBIND11_DEBUG_MARKER) -# define _DEBUG -# undef PYBIND11_DEBUG_MARKER -# endif -# pragma warning(pop) +# if defined(PYBIND11_DEBUG_MARKER) +# define _DEBUG +# undef PYBIND11_DEBUG_MARKER +# endif +PYBIND11_WARNING_POP #endif #include <cstddef> #include <cstring> -#include <forward_list> -#include <vector> -#include <string> -#include <stdexcept> #include <exception> -#include <unordered_set> -#include <unordered_map> +#include <forward_list> #include <memory> -#include <typeindex> +#include <stdexcept> +#include <string> #include <type_traits> +#include <typeindex> +#include <unordered_map> +#include <unordered_set> +#include <vector> +#if defined(__has_include) +# if __has_include(<version>) +# include <version> +# endif +#endif -#if PY_MAJOR_VERSION >= 3 /// Compatibility macros for various Python versions +// Must be after including <version> or one of the other headers specified by the standard +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + +// See description of PR #4246: +#if !defined(NDEBUG) && !defined(PY_ASSERT_GIL_HELD_INCREF_DECREF) \ + && !(defined(PYPY_VERSION) \ + && defined(_MSC_VER)) /* PyPy Windows: pytest hangs indefinitely at the end of the \ + process (see PR #4268) */ \ + && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +# define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF +#endif + +// #define PYBIND11_STR_LEGACY_PERMISSIVE +// If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject +// (probably surprising and never documented, but this was the +// legacy behavior until and including v2.6.x). As a side-effect, +// pybind11::isinstance<str>() is true for both pybind11::str and +// pybind11::bytes. +// If UNDEFINED, pybind11::str can only hold PyUnicodeObject, and +// pybind11::isinstance<str>() is true only for pybind11::str. +// However, for Python 2 only (!), the pybind11::str caster +// implicitly decoded bytes to PyUnicodeObject. This was to ease +// the transition from the legacy behavior to the non-permissive +// behavior. + +/// Compatibility macros for Python 2 / Python 3 versions TODO: remove #define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyInstanceMethod_New(ptr) #define PYBIND11_INSTANCE_METHOD_CHECK PyInstanceMethod_Check #define PYBIND11_INSTANCE_METHOD_GET_FUNCTION PyInstanceMethod_GET_FUNCTION @@ -175,8 +357,8 @@ #define PYBIND11_BYTES_SIZE PyBytes_Size #define PYBIND11_LONG_CHECK(o) PyLong_Check(o) #define PYBIND11_LONG_AS_LONGLONG(o) PyLong_AsLongLong(o) -#define PYBIND11_LONG_FROM_SIGNED(o) PyLong_FromSsize_t((ssize_t) o) -#define PYBIND11_LONG_FROM_UNSIGNED(o) PyLong_FromSize_t((size_t) o) +#define PYBIND11_LONG_FROM_SIGNED(o) PyLong_FromSsize_t((ssize_t) (o)) +#define PYBIND11_LONG_FROM_UNSIGNED(o) PyLong_FromSize_t((size_t) (o)) #define PYBIND11_BYTES_NAME "bytes" #define PYBIND11_STRING_NAME "str" #define PYBIND11_SLICE_OBJECT PyObject @@ -187,81 +369,42 @@ #define PYBIND11_BUILTINS_MODULE "builtins" // Providing a separate declaration to make Clang's -Wmissing-prototypes happy. // See comment for PYBIND11_MODULE below for why this is marked "maybe unused". -#define PYBIND11_PLUGIN_IMPL(name) \ - extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT PyObject *PyInit_##name(); \ +#define PYBIND11_PLUGIN_IMPL(name) \ + extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT PyObject *PyInit_##name(); \ extern "C" PYBIND11_EXPORT PyObject *PyInit_##name() -#else -#define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyMethod_New(ptr, nullptr, class_) -#define PYBIND11_INSTANCE_METHOD_CHECK PyMethod_Check -#define PYBIND11_INSTANCE_METHOD_GET_FUNCTION PyMethod_GET_FUNCTION -#define PYBIND11_BYTES_CHECK PyString_Check -#define PYBIND11_BYTES_FROM_STRING PyString_FromString -#define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyString_FromStringAndSize -#define PYBIND11_BYTES_AS_STRING_AND_SIZE PyString_AsStringAndSize -#define PYBIND11_BYTES_AS_STRING PyString_AsString -#define PYBIND11_BYTES_SIZE PyString_Size -#define PYBIND11_LONG_CHECK(o) (PyInt_Check(o) || PyLong_Check(o)) -#define PYBIND11_LONG_AS_LONGLONG(o) (PyInt_Check(o) ? (long long) PyLong_AsLong(o) : PyLong_AsLongLong(o)) -#define PYBIND11_LONG_FROM_SIGNED(o) PyInt_FromSsize_t((ssize_t) o) // Returns long if needed. -#define PYBIND11_LONG_FROM_UNSIGNED(o) PyInt_FromSize_t((size_t) o) // Returns long if needed. -#define PYBIND11_BYTES_NAME "str" -#define PYBIND11_STRING_NAME "unicode" -#define PYBIND11_SLICE_OBJECT PySliceObject -#define PYBIND11_FROM_STRING PyString_FromString -#define PYBIND11_STR_TYPE ::pybind11::bytes -#define PYBIND11_BOOL_ATTR "__nonzero__" -#define PYBIND11_NB_BOOL(ptr) ((ptr)->nb_nonzero) -#define PYBIND11_BUILTINS_MODULE "__builtin__" -// Providing a separate PyInit decl to make Clang's -Wmissing-prototypes happy. -// See comment for PYBIND11_MODULE below for why this is marked "maybe unused". -#define PYBIND11_PLUGIN_IMPL(name) \ - static PyObject *pybind11_init_wrapper(); \ - extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT void init##name(); \ - extern "C" PYBIND11_EXPORT void init##name() { \ - (void)pybind11_init_wrapper(); \ - } \ - PyObject *pybind11_init_wrapper() -#endif - -#if PY_VERSION_HEX >= 0x03050000 && PY_VERSION_HEX < 0x03050200 -extern "C" { - struct _Py_atomic_address { void *value; }; - PyAPI_DATA(_Py_atomic_address) _PyThreadState_Current; -} -#endif - #define PYBIND11_TRY_NEXT_OVERLOAD ((PyObject *) 1) // special failure return code #define PYBIND11_STRINGIFY(x) #x #define PYBIND11_TOSTRING(x) PYBIND11_STRINGIFY(x) #define PYBIND11_CONCAT(first, second) first##second -#define PYBIND11_ENSURE_INTERNALS_READY \ - pybind11::detail::get_internals(); - -#define PYBIND11_CHECK_PYTHON_VERSION \ - { \ - const char *compiled_ver = PYBIND11_TOSTRING(PY_MAJOR_VERSION) \ - "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ - const char *runtime_ver = Py_GetVersion(); \ - size_t len = std::strlen(compiled_ver); \ - if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ - || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ - PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for Python %s, " \ - "but the interpreter version is incompatible: %s.", \ - compiled_ver, runtime_ver); \ - return nullptr; \ - } \ +#define PYBIND11_ENSURE_INTERNALS_READY pybind11::detail::get_internals(); + +#define PYBIND11_CHECK_PYTHON_VERSION \ + { \ + const char *compiled_ver \ + = PYBIND11_TOSTRING(PY_MAJOR_VERSION) "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ + const char *runtime_ver = Py_GetVersion(); \ + size_t len = std::strlen(compiled_ver); \ + if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ + || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ + PyErr_Format(PyExc_ImportError, \ + "Python version mismatch: module was compiled for Python %s, " \ + "but the interpreter version is incompatible: %s.", \ + compiled_ver, \ + runtime_ver); \ + return nullptr; \ + } \ } -#define PYBIND11_CATCH_INIT_EXCEPTIONS \ - catch (pybind11::error_already_set &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ - return nullptr; \ - } \ +#define PYBIND11_CATCH_INIT_EXCEPTIONS \ + catch (pybind11::error_already_set & e) { \ + pybind11::raise_from(e, PyExc_ImportError, "initialization failed"); \ + return nullptr; \ + } \ + catch (const std::exception &e) { \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ + } /** \rst ***Deprecated in favor of PYBIND11_MODULE*** @@ -278,21 +421,22 @@ extern "C" { return m.ptr(); } \endrst */ -#define PYBIND11_PLUGIN(name) \ - PYBIND11_DEPRECATED("PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE") \ - static PyObject *pybind11_init(); \ - PYBIND11_PLUGIN_IMPL(name) { \ - PYBIND11_CHECK_PYTHON_VERSION \ - PYBIND11_ENSURE_INTERNALS_READY \ - try { \ - return pybind11_init(); \ - } PYBIND11_CATCH_INIT_EXCEPTIONS \ - } \ +#define PYBIND11_PLUGIN(name) \ + PYBIND11_DEPRECATED("PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE") \ + static PyObject *pybind11_init(); \ + PYBIND11_PLUGIN_IMPL(name) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_ENSURE_INTERNALS_READY \ + try { \ + return pybind11_init(); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + } \ PyObject *pybind11_init() /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. @@ -311,29 +455,34 @@ extern "C" { }); } \endrst */ -#define PYBIND11_MODULE(name, variable) \ - static ::pybind11::module_::module_def \ - PYBIND11_CONCAT(pybind11_module_def_, name) PYBIND11_MAYBE_UNUSED; \ - PYBIND11_MAYBE_UNUSED \ - static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ - PYBIND11_PLUGIN_IMPL(name) { \ - PYBIND11_CHECK_PYTHON_VERSION \ - PYBIND11_ENSURE_INTERNALS_READY \ - auto m = ::pybind11::module_::create_extension_module( \ - PYBIND11_TOSTRING(name), nullptr, \ - &PYBIND11_CONCAT(pybind11_module_def_, name)); \ - try { \ - PYBIND11_CONCAT(pybind11_init_, name)(m); \ - return m.ptr(); \ - } PYBIND11_CATCH_INIT_EXCEPTIONS \ - } \ - void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &variable) - +#define PYBIND11_MODULE(name, variable) \ + static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name) \ + PYBIND11_MAYBE_UNUSED; \ + PYBIND11_MAYBE_UNUSED \ + static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ + PYBIND11_PLUGIN_IMPL(name) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_ENSURE_INTERNALS_READY \ + auto m = ::pybind11::module_::create_extension_module( \ + PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name)); \ + try { \ + PYBIND11_CONCAT(pybind11_init_, name)(m); \ + return m.ptr(); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + } \ + void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ & (variable)) PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ssize_t = Py_ssize_t; -using size_t = std::size_t; +using size_t = std::size_t; + +template <typename IntType> +inline ssize_t ssize_t_cast(const IntType &val) { + static_assert(sizeof(IntType) <= sizeof(ssize_t), "Implicit narrowing is not permitted."); + return static_cast<ssize_t>(val); +} /// Approach used to cast a previously unknown C++ instance into a Python object enum class return_value_policy : uint8_t { @@ -352,7 +501,7 @@ enum class return_value_policy : uint8_t { /** Reference an existing object (i.e. do not create a new copy) and take ownership. Python will call the destructor and delete operator when the - object’s reference count reaches zero. Undefined behavior ensues when + object's reference count reaches zero. Undefined behavior ensues when the C++ side does the same.. */ take_ownership, @@ -368,7 +517,7 @@ enum class return_value_policy : uint8_t { move, /** Reference an existing object, but do not take ownership. The C++ side - is responsible for managing the object’s lifetime and deallocating it + is responsible for managing the object's lifetime and deallocating it when it is no longer used. Warning: undefined behavior will ensue when the C++ side deletes an object that is still referenced and used by Python. */ @@ -377,7 +526,7 @@ enum class return_value_policy : uint8_t { /** This policy only applies to methods and properties. It references the object without taking ownership similar to the above return_value_policy::reference policy. In contrast to that policy, the - function or property’s implicit this argument (called the parent) is + function or property's implicit this argument (called the parent) is considered to be the the owner of the return value (the child). pybind11 then couples the lifetime of the parent to the child via a reference relationship that ensures that the parent cannot be garbage @@ -389,10 +538,14 @@ enum class return_value_policy : uint8_t { PYBIND11_NAMESPACE_BEGIN(detail) -inline static constexpr int log2(size_t n, int k = 0) { return (n <= 1) ? k : log2(n >> 1, k + 1); } +inline static constexpr int log2(size_t n, int k = 0) { + return (n <= 1) ? k : log2(n >> 1, k + 1); +} // Returns the size as a multiple of sizeof(void *), rounded up. -inline static constexpr size_t size_in_ptrs(size_t s) { return 1 + ((s - 1) >> log2(sizeof(void *))); } +inline static constexpr size_t size_in_ptrs(size_t s) { + return 1 + ((s - 1) >> log2(sizeof(void *))); +} /** * The space to allocate for simple layout instance holders (see below) in multiple of the size of @@ -402,7 +555,7 @@ inline static constexpr size_t size_in_ptrs(size_t s) { return 1 + ((s - 1) >> l */ constexpr size_t instance_simple_holder_in_ptrs() { static_assert(sizeof(std::shared_ptr<int>) >= sizeof(std::unique_ptr<int>), - "pybind assumes std::shared_ptrs are at least as big as std::unique_ptrs"); + "pybind assumes std::shared_ptrs are at least as big as std::unique_ptrs"); return size_in_ptrs(sizeof(std::shared_ptr<int>)); } @@ -430,21 +583,21 @@ struct instance { /** * An instance has two possible value/holder layouts. * - * Simple layout (when this flag is true), means the `simple_value_holder` is set with a pointer - * and the holder object governing that pointer, i.e. [val1*][holder]. This layout is applied - * whenever there is no python-side multiple inheritance of bound C++ types *and* the type's - * holder will fit in the default space (which is large enough to hold either a std::unique_ptr - * or std::shared_ptr). + * Simple layout (when this flag is true), means the `simple_value_holder` is set with a + * pointer and the holder object governing that pointer, i.e. [val1*][holder]. This layout is + * applied whenever there is no python-side multiple inheritance of bound C++ types *and* the + * type's holder will fit in the default space (which is large enough to hold either a + * std::unique_ptr or std::shared_ptr). * - * Non-simple layout applies when using custom holders that require more space than `shared_ptr` - * (which is typically the size of two pointers), or when multiple inheritance is used on the - * python side. Non-simple layout allocates the required amount of memory to have multiple - * bound C++ classes as parents. Under this layout, `nonsimple.values_and_holders` is set to a - * pointer to allocated space of the required space to hold a sequence of value pointers and - * holders followed `status`, a set of bit flags (1 byte each), i.e. - * [val1*][holder1][val2*][holder2]...[bb...] where each [block] is rounded up to a multiple of - * `sizeof(void *)`. `nonsimple.status` is, for convenience, a pointer to the - * beginning of the [bb...] block (but not independently allocated). + * Non-simple layout applies when using custom holders that require more space than + * `shared_ptr` (which is typically the size of two pointers), or when multiple inheritance is + * used on the python side. Non-simple layout allocates the required amount of memory to have + * multiple bound C++ classes as parents. Under this layout, `nonsimple.values_and_holders` is + * set to a pointer to allocated space of the required space to hold a sequence of value + * pointers and holders followed `status`, a set of bit flags (1 byte each), i.e. + * [val1*][holder1][val2*][holder2]...[bb...] where each [block] is rounded up to a multiple + * of `sizeof(void *)`. `nonsimple.status` is, for convenience, a pointer to the beginning of + * the [bb...] block (but not independently allocated). * * Status bits indicate whether the associated holder is constructed (& * status_holder_constructed) and whether the value pointer is registered (& @@ -458,7 +611,8 @@ struct instance { /// If true, get_internals().patients has an entry for this object bool has_patients : 1; - /// Initializes all of the above type/values/holders data (but not the instance values themselves) + /// Initializes all of the above type/values/holders data (but not the instance values + /// themselves) void allocate_layout(); /// Destroys/deallocates all of the above @@ -467,141 +621,247 @@ struct instance { /// Returns the value_and_holder wrapper for the given type (or the first, if `find_type` /// omitted). Returns a default-constructed (with `.inst = nullptr`) object on failure if /// `throw_if_missing` is false. - value_and_holder get_value_and_holder(const type_info *find_type = nullptr, bool throw_if_missing = true); + value_and_holder get_value_and_holder(const type_info *find_type = nullptr, + bool throw_if_missing = true); /// Bit values for the non-simple status flags - static constexpr uint8_t status_holder_constructed = 1; + static constexpr uint8_t status_holder_constructed = 1; static constexpr uint8_t status_instance_registered = 2; }; -static_assert(std::is_standard_layout<instance>::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); +static_assert(std::is_standard_layout<instance>::value, + "Internal error: `pybind11::detail::instance` is not standard layout!"); /// from __cpp_future__ import (convenient aliases from C++14/17) -#if defined(PYBIND11_CPP14) && (!defined(_MSC_VER) || _MSC_VER >= 1910) -using std::enable_if_t; +#if defined(PYBIND11_CPP14) using std::conditional_t; +using std::enable_if_t; using std::remove_cv_t; using std::remove_reference_t; #else -template <bool B, typename T = void> using enable_if_t = typename std::enable_if<B, T>::type; -template <bool B, typename T, typename F> using conditional_t = typename std::conditional<B, T, F>::type; -template <typename T> using remove_cv_t = typename std::remove_cv<T>::type; -template <typename T> using remove_reference_t = typename std::remove_reference<T>::type; +template <bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; +template <bool B, typename T, typename F> +using conditional_t = typename std::conditional<B, T, F>::type; +template <typename T> +using remove_cv_t = typename std::remove_cv<T>::type; +template <typename T> +using remove_reference_t = typename std::remove_reference<T>::type; +#endif + +#if defined(PYBIND11_CPP20) +using std::remove_cvref; +using std::remove_cvref_t; +#else +template <class T> +struct remove_cvref { + using type = remove_cv_t<remove_reference_t<T>>; +}; +template <class T> +using remove_cvref_t = typename remove_cvref<T>::type; #endif +/// Example usage: is_same_ignoring_cvref<T, PyObject *>::value +template <typename T, typename U> +using is_same_ignoring_cvref = std::is_same<detail::remove_cvref_t<T>, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; using std::make_index_sequence; #else -template<size_t ...> struct index_sequence { }; -template<size_t N, size_t ...S> struct make_index_sequence_impl : make_index_sequence_impl <N - 1, N - 1, S...> { }; -template<size_t ...S> struct make_index_sequence_impl <0, S...> { using type = index_sequence<S...>; }; -template<size_t N> using make_index_sequence = typename make_index_sequence_impl<N>::type; +template <size_t...> +struct index_sequence {}; +template <size_t N, size_t... S> +struct make_index_sequence_impl : make_index_sequence_impl<N - 1, N - 1, S...> {}; +template <size_t... S> +struct make_index_sequence_impl<0, S...> { + using type = index_sequence<S...>; +}; +template <size_t N> +using make_index_sequence = typename make_index_sequence_impl<N>::type; #endif /// Make an index sequence of the indices of true arguments -template <typename ISeq, size_t, bool...> struct select_indices_impl { using type = ISeq; }; -template <size_t... IPrev, size_t I, bool B, bool... Bs> struct select_indices_impl<index_sequence<IPrev...>, I, B, Bs...> - : select_indices_impl<conditional_t<B, index_sequence<IPrev..., I>, index_sequence<IPrev...>>, I + 1, Bs...> {}; -template <bool... Bs> using select_indices = typename select_indices_impl<index_sequence<>, 0, Bs...>::type; +template <typename ISeq, size_t, bool...> +struct select_indices_impl { + using type = ISeq; +}; +template <size_t... IPrev, size_t I, bool B, bool... Bs> +struct select_indices_impl<index_sequence<IPrev...>, I, B, Bs...> + : select_indices_impl<conditional_t<B, index_sequence<IPrev..., I>, index_sequence<IPrev...>>, + I + 1, + Bs...> {}; +template <bool... Bs> +using select_indices = typename select_indices_impl<index_sequence<>, 0, Bs...>::type; /// Backports of std::bool_constant and std::negation to accommodate older compilers -template <bool B> using bool_constant = std::integral_constant<bool, B>; -template <typename T> struct negation : bool_constant<!T::value> { }; +template <bool B> +using bool_constant = std::integral_constant<bool, B>; +template <typename T> +struct negation : bool_constant<!T::value> {}; // PGI/Intel cannot detect operator delete with the "compatible" void_t impl, so // using the new one (C++14 defect, so generally works on newer compilers, even // if not in C++17 mode) #if defined(__PGIC__) || defined(__INTEL_COMPILER) -template<typename... > using void_t = void; +template <typename...> +using void_t = void; #else -template <typename...> struct void_t_impl { using type = void; }; -template <typename... Ts> using void_t = typename void_t_impl<Ts...>::type; +template <typename...> +struct void_t_impl { + using type = void; +}; +template <typename... Ts> +using void_t = typename void_t_impl<Ts...>::type; #endif - /// Compile-time all/any/none of that check the boolean value of all template types #if defined(__cpp_fold_expressions) && !(defined(_MSC_VER) && (_MSC_VER < 1916)) -template <class... Ts> using all_of = bool_constant<(Ts::value && ...)>; -template <class... Ts> using any_of = bool_constant<(Ts::value || ...)>; +template <class... Ts> +using all_of = bool_constant<(Ts::value && ...)>; +template <class... Ts> +using any_of = bool_constant<(Ts::value || ...)>; #elif !defined(_MSC_VER) -template <bool...> struct bools {}; -template <class... Ts> using all_of = std::is_same< - bools<Ts::value..., true>, - bools<true, Ts::value...>>; -template <class... Ts> using any_of = negation<all_of<negation<Ts>...>>; +template <bool...> +struct bools {}; +template <class... Ts> +using all_of = std::is_same<bools<Ts::value..., true>, bools<true, Ts::value...>>; +template <class... Ts> +using any_of = negation<all_of<negation<Ts>...>>; #else // MSVC has trouble with the above, but supports std::conjunction, which we can use instead (albeit // at a slight loss of compilation efficiency). -template <class... Ts> using all_of = std::conjunction<Ts...>; -template <class... Ts> using any_of = std::disjunction<Ts...>; +template <class... Ts> +using all_of = std::conjunction<Ts...>; +template <class... Ts> +using any_of = std::disjunction<Ts...>; #endif -template <class... Ts> using none_of = negation<any_of<Ts...>>; +template <class... Ts> +using none_of = negation<any_of<Ts...>>; -template <class T, template<class> class... Predicates> using satisfies_all_of = all_of<Predicates<T>...>; -template <class T, template<class> class... Predicates> using satisfies_any_of = any_of<Predicates<T>...>; -template <class T, template<class> class... Predicates> using satisfies_none_of = none_of<Predicates<T>...>; +template <class T, template <class> class... Predicates> +using satisfies_all_of = all_of<Predicates<T>...>; +template <class T, template <class> class... Predicates> +using satisfies_any_of = any_of<Predicates<T>...>; +template <class T, template <class> class... Predicates> +using satisfies_none_of = none_of<Predicates<T>...>; /// Strip the class from a method type -template <typename T> struct remove_class { }; -template <typename C, typename R, typename... A> struct remove_class<R (C::*)(A...)> { using type = R (A...); }; -template <typename C, typename R, typename... A> struct remove_class<R (C::*)(A...) const> { using type = R (A...); }; - +template <typename T> +struct remove_class {}; +template <typename C, typename R, typename... A> +struct remove_class<R (C::*)(A...)> { + using type = R(A...); +}; +template <typename C, typename R, typename... A> +struct remove_class<R (C::*)(A...) const> { + using type = R(A...); +}; +#ifdef __cpp_noexcept_function_type +template <typename C, typename R, typename... A> +struct remove_class<R (C::*)(A...) noexcept> { + using type = R(A...); +}; +template <typename C, typename R, typename... A> +struct remove_class<R (C::*)(A...) const noexcept> { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers -template <typename T> struct intrinsic_type { using type = T; }; -template <typename T> struct intrinsic_type<const T> { using type = typename intrinsic_type<T>::type; }; -template <typename T> struct intrinsic_type<T*> { using type = typename intrinsic_type<T>::type; }; -template <typename T> struct intrinsic_type<T&> { using type = typename intrinsic_type<T>::type; }; -template <typename T> struct intrinsic_type<T&&> { using type = typename intrinsic_type<T>::type; }; -template <typename T, size_t N> struct intrinsic_type<const T[N]> { using type = typename intrinsic_type<T>::type; }; -template <typename T, size_t N> struct intrinsic_type<T[N]> { using type = typename intrinsic_type<T>::type; }; -template <typename T> using intrinsic_t = typename intrinsic_type<T>::type; +template <typename T> +struct intrinsic_type { + using type = T; +}; +template <typename T> +struct intrinsic_type<const T> { + using type = typename intrinsic_type<T>::type; +}; +template <typename T> +struct intrinsic_type<T *> { + using type = typename intrinsic_type<T>::type; +}; +template <typename T> +struct intrinsic_type<T &> { + using type = typename intrinsic_type<T>::type; +}; +template <typename T> +struct intrinsic_type<T &&> { + using type = typename intrinsic_type<T>::type; +}; +template <typename T, size_t N> +struct intrinsic_type<const T[N]> { + using type = typename intrinsic_type<T>::type; +}; +template <typename T, size_t N> +struct intrinsic_type<T[N]> { + using type = typename intrinsic_type<T>::type; +}; +template <typename T> +using intrinsic_t = typename intrinsic_type<T>::type; /// Helper type to replace 'void' in some expressions -struct void_type { }; +struct void_type {}; /// Helper template which holds a list of types -template <typename...> struct type_list { }; +template <typename...> +struct type_list {}; /// Compile-time integer sum #ifdef __cpp_fold_expressions -template <typename... Ts> constexpr size_t constexpr_sum(Ts... ns) { return (0 + ... + size_t{ns}); } +template <typename... Ts> +constexpr size_t constexpr_sum(Ts... ns) { + return (0 + ... + size_t{ns}); +} #else constexpr size_t constexpr_sum() { return 0; } template <typename T, typename... Ts> -constexpr size_t constexpr_sum(T n, Ts... ns) { return size_t{n} + constexpr_sum(ns...); } +constexpr size_t constexpr_sum(T n, Ts... ns) { + return size_t{n} + constexpr_sum(ns...); +} #endif PYBIND11_NAMESPACE_BEGIN(constexpr_impl) /// Implementation details for constexpr functions constexpr int first(int i) { return i; } template <typename T, typename... Ts> -constexpr int first(int i, T v, Ts... vs) { return v ? i : first(i + 1, vs...); } +constexpr int first(int i, T v, Ts... vs) { + return v ? i : first(i + 1, vs...); +} constexpr int last(int /*i*/, int result) { return result; } template <typename T, typename... Ts> -constexpr int last(int i, int result, T v, Ts... vs) { return last(i + 1, v ? i : result, vs...); } +constexpr int last(int i, int result, T v, Ts... vs) { + return last(i + 1, v ? i : result, vs...); +} PYBIND11_NAMESPACE_END(constexpr_impl) -/// Return the index of the first type in Ts which satisfies Predicate<T>. Returns sizeof...(Ts) if -/// none match. -template <template<typename> class Predicate, typename... Ts> -constexpr int constexpr_first() { return constexpr_impl::first(0, Predicate<Ts>::value...); } +/// Return the index of the first type in Ts which satisfies Predicate<T>. +/// Returns sizeof...(Ts) if none match. +template <template <typename> class Predicate, typename... Ts> +constexpr int constexpr_first() { + return constexpr_impl::first(0, Predicate<Ts>::value...); +} /// Return the index of the last type in Ts which satisfies Predicate<T>, or -1 if none match. -template <template<typename> class Predicate, typename... Ts> -constexpr int constexpr_last() { return constexpr_impl::last(0, -1, Predicate<Ts>::value...); } +template <template <typename> class Predicate, typename... Ts> +constexpr int constexpr_last() { + return constexpr_impl::last(0, -1, Predicate<Ts>::value...); +} /// Return the Nth element from the parameter pack template <size_t N, typename T, typename... Ts> -struct pack_element { using type = typename pack_element<N - 1, Ts...>::type; }; +struct pack_element { + using type = typename pack_element<N - 1, Ts...>::type; +}; template <typename T, typename... Ts> -struct pack_element<0, T, Ts...> { using type = T; }; +struct pack_element<0, T, Ts...> { + using type = T; +}; /// Return the one and only type which matches the predicate, or Default if none match. /// If more than one type matches the predicate, fail at compile-time. -template <template<typename> class Predicate, typename Default, typename... Ts> +template <template <typename> class Predicate, typename Default, typename... Ts> struct exactly_one { static constexpr auto found = constexpr_sum(Predicate<Ts>::value...); static_assert(found <= 1, "Found more than one type matching the predicate"); @@ -609,62 +869,83 @@ struct exactly_one { static constexpr auto index = found ? constexpr_first<Predicate, Ts...>() : 0; using type = conditional_t<found, typename pack_element<index, Ts...>::type, Default>; }; -template <template<typename> class P, typename Default> -struct exactly_one<P, Default> { using type = Default; }; +template <template <typename> class P, typename Default> +struct exactly_one<P, Default> { + using type = Default; +}; -template <template<typename> class Predicate, typename Default, typename... Ts> +template <template <typename> class Predicate, typename Default, typename... Ts> using exactly_one_t = typename exactly_one<Predicate, Default, Ts...>::type; /// Defer the evaluation of type T until types Us are instantiated -template <typename T, typename... /*Us*/> struct deferred_type { using type = T; }; -template <typename T, typename... Us> using deferred_t = typename deferred_type<T, Us...>::type; +template <typename T, typename... /*Us*/> +struct deferred_type { + using type = T; +}; +template <typename T, typename... Us> +using deferred_t = typename deferred_type<T, Us...>::type; /// Like is_base_of, but requires a strict base (i.e. `is_strict_base_of<T, T>::value == false`, /// unlike `std::is_base_of`) -template <typename Base, typename Derived> using is_strict_base_of = bool_constant< - std::is_base_of<Base, Derived>::value && !std::is_same<Base, Derived>::value>; - -/// Like is_base_of, but also requires that the base type is accessible (i.e. that a Derived pointer -/// can be converted to a Base pointer) -/// For unions, `is_base_of<T, T>::value` is False, so we need to check `is_same` as well. -template <typename Base, typename Derived> using is_accessible_base_of = bool_constant< - (std::is_same<Base, Derived>::value || std::is_base_of<Base, Derived>::value) && std::is_convertible<Derived *, Base *>::value>; - -template <template<typename...> class Base> +template <typename Base, typename Derived> +using is_strict_base_of + = bool_constant<std::is_base_of<Base, Derived>::value && !std::is_same<Base, Derived>::value>; + +/// Like is_base_of, but also requires that the base type is accessible (i.e. that a Derived +/// pointer can be converted to a Base pointer) For unions, `is_base_of<T, T>::value` is False, so +/// we need to check `is_same` as well. +template <typename Base, typename Derived> +using is_accessible_base_of + = bool_constant<(std::is_same<Base, Derived>::value || std::is_base_of<Base, Derived>::value) + && std::is_convertible<Derived *, Base *>::value>; + +template <template <typename...> class Base> struct is_template_base_of_impl { - template <typename... Us> static std::true_type check(Base<Us...> *); + template <typename... Us> + static std::true_type check(Base<Us...> *); static std::false_type check(...); }; /// Check if a template is the base of a type. For example: /// `is_template_base_of<Base, T>` is true if `struct T : Base<U> {}` where U can be anything -template <template<typename...> class Base, typename T> +template <template <typename...> class Base, typename T> +// Sadly, all MSVC versions incl. 2022 need the workaround, even in C++20 mode. +// See also: https://github.com/pybind/pybind11/pull/3741 #if !defined(_MSC_VER) -using is_template_base_of = decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T>*)nullptr)); -#else // MSVC2015 has trouble with decltype in template aliases -struct is_template_base_of : decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T>*)nullptr)) { }; +using is_template_base_of + = decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T> *) nullptr)); +#else +struct is_template_base_of + : decltype(is_template_base_of_impl<Base>::check((intrinsic_t<T> *) nullptr)) { +}; #endif /// Check if T is an instantiation of the template `Class`. For example: /// `is_instantiation<shared_ptr, T>` is true if `T == shared_ptr<U>` where U can be anything. -template <template<typename...> class Class, typename T> -struct is_instantiation : std::false_type { }; -template <template<typename...> class Class, typename... Us> -struct is_instantiation<Class, Class<Us...>> : std::true_type { }; +template <template <typename...> class Class, typename T> +struct is_instantiation : std::false_type {}; +template <template <typename...> class Class, typename... Us> +struct is_instantiation<Class, Class<Us...>> : std::true_type {}; /// Check if T is std::shared_ptr<U> where U can be anything -template <typename T> using is_shared_ptr = is_instantiation<std::shared_ptr, T>; +template <typename T> +using is_shared_ptr = is_instantiation<std::shared_ptr, T>; /// Check if T looks like an input iterator -template <typename T, typename = void> struct is_input_iterator : std::false_type {}; +template <typename T, typename = void> +struct is_input_iterator : std::false_type {}; template <typename T> -struct is_input_iterator<T, void_t<decltype(*std::declval<T &>()), decltype(++std::declval<T &>())>> +struct is_input_iterator<T, + void_t<decltype(*std::declval<T &>()), decltype(++std::declval<T &>())>> : std::true_type {}; -template <typename T> using is_function_pointer = bool_constant< - std::is_pointer<T>::value && std::is_function<typename std::remove_pointer<T>::type>::value>; +template <typename T> +using is_function_pointer + = bool_constant<std::is_pointer<T>::value + && std::is_function<typename std::remove_pointer<T>::type>::value>; -template <typename F> struct strip_function_object { +template <typename F> +struct strip_function_object { // If you are encountering an // 'error: name followed by "::" must be a class or namespace name' // with the Intel compiler and a noexcept function here, @@ -677,47 +958,46 @@ template <typename Function, typename F = remove_reference_t<Function>> using function_signature_t = conditional_t< std::is_function<F>::value, F, - typename conditional_t< - std::is_pointer<F>::value || std::is_member_pointer<F>::value, - std::remove_pointer<F>, - strip_function_object<F> - >::type ->; + typename conditional_t<std::is_pointer<F>::value || std::is_member_pointer<F>::value, + std::remove_pointer<F>, + strip_function_object<F>>::type>; /// Returns true if the type looks like a lambda: that is, isn't a function, pointer or member /// pointer. Note that this can catch all sorts of other things, too; this is intended to be used /// in a place where passing a lambda makes sense. -template <typename T> using is_lambda = satisfies_none_of<remove_reference_t<T>, - std::is_function, std::is_pointer, std::is_member_pointer>; - -/// Ignore that a variable is unused in compiler warnings -inline void ignore_unused(const int *) { } +template <typename T> +using is_lambda = satisfies_none_of<remove_reference_t<T>, + std::is_function, + std::is_pointer, + std::is_member_pointer>; // [workaround(intel)] Internal error on fold expression /// Apply a function over each element of a parameter pack #if defined(__cpp_fold_expressions) && !defined(__INTEL_COMPILER) // Intel compiler produces an internal error on this fold expression (tested with ICC 19.0.2) -#define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) (((PATTERN), void()), ...) +# define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) (((PATTERN), void()), ...) #else using expand_side_effects = bool[]; -#define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) (void)pybind11::detail::expand_side_effects{ ((PATTERN), void(), false)..., false } +# define PYBIND11_EXPAND_SIDE_EFFECTS(PATTERN) \ + (void) pybind11::detail::expand_side_effects { ((PATTERN), void(), false)..., false } #endif PYBIND11_NAMESPACE_END(detail) /// C++ bindings of builtin Python exceptions -class builtin_exception : public std::runtime_error { +class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { public: using std::runtime_error::runtime_error; /// Set the error using the Python C API virtual void set_error() const = 0; }; -#define PYBIND11_RUNTIME_EXCEPTION(name, type) \ - class name : public builtin_exception { public: \ - using builtin_exception::builtin_exception; \ - name() : name("") { } \ - void set_error() const override { PyErr_SetString(type, what()); } \ +#define PYBIND11_RUNTIME_EXCEPTION(name, type) \ + class PYBIND11_EXPORT_EXCEPTION name : public builtin_exception { \ + public: \ + using builtin_exception::builtin_exception; \ + name() : name("") {} \ + void set_error() const override { PyErr_SetString(type, what()); } \ }; PYBIND11_RUNTIME_EXCEPTION(stop_iteration, PyExc_StopIteration) @@ -727,13 +1007,32 @@ PYBIND11_RUNTIME_EXCEPTION(value_error, PyExc_ValueError) PYBIND11_RUNTIME_EXCEPTION(type_error, PyExc_TypeError) PYBIND11_RUNTIME_EXCEPTION(buffer_error, PyExc_BufferError) PYBIND11_RUNTIME_EXCEPTION(import_error, PyExc_ImportError) -PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybind11::cast or handle::call fail due to a type casting error +PYBIND11_RUNTIME_EXCEPTION(attribute_error, PyExc_AttributeError) +PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybind11::cast or + /// handle::call fail due to a type + /// casting error PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally -[[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const char *reason) { throw std::runtime_error(reason); } -[[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); } +[[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const char *reason) { + assert(!PyErr_Occurred()); + throw std::runtime_error(reason); +} +[[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const std::string &reason) { + assert(!PyErr_Occurred()); + throw std::runtime_error(reason); +} -template <typename T, typename SFINAE = void> struct format_descriptor { }; +template <typename T, typename SFINAE = void> +struct format_descriptor {}; + +template <typename T> +struct format_descriptor< + T, + detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h @@ -741,25 +1040,38 @@ PYBIND11_NAMESPACE_BEGIN(detail) // complex float,double,long double. Note that the long double types only participate when long // double is actually longer than double (it isn't under MSVC). // NB: not only the string below but also complex.h and numpy.h rely on this order. -template <typename T, typename SFINAE = void> struct is_fmt_numeric { static constexpr bool value = false; }; -template <typename T> struct is_fmt_numeric<T, enable_if_t<std::is_arithmetic<T>::value>> { +template <typename T, typename SFINAE = void> +struct is_fmt_numeric { + static constexpr bool value = false; +}; +template <typename T> +struct is_fmt_numeric<T, enable_if_t<std::is_arithmetic<T>::value>> { static constexpr bool value = true; - static constexpr int index = std::is_same<T, bool>::value ? 0 : 1 + ( - std::is_integral<T>::value ? detail::log2(sizeof(T))*2 + std::is_unsigned<T>::value : 8 + ( - std::is_same<T, double>::value ? 1 : std::is_same<T, long double>::value ? 2 : 0)); + static constexpr int index + = std::is_same<T, bool>::value + ? 0 + : 1 + + (std::is_integral<T>::value + ? detail::log2(sizeof(T)) * 2 + std::is_unsigned<T>::value + : 8 + + (std::is_same<T, double>::value ? 1 + : std::is_same<T, long double>::value ? 2 + : 0)); }; PYBIND11_NAMESPACE_END(detail) -template <typename T> struct format_descriptor<T, detail::enable_if_t<std::is_arithmetic<T>::value>> { +template <typename T> +struct format_descriptor<T, detail::enable_if_t<std::is_arithmetic<T>::value>> { static constexpr const char c = "?bBhHiIqQfdg"[detail::is_fmt_numeric<T>::index]; - static constexpr const char value[2] = { c, '\0' }; + static constexpr const char value[2] = {c, '\0'}; static std::string format() { return std::string(1, c); } }; #if !defined(PYBIND11_CPP17) -template <typename T> constexpr const char format_descriptor< - T, detail::enable_if_t<std::is_arithmetic<T>::value>>::value[2]; +template <typename T> +constexpr const char + format_descriptor<T, detail::enable_if_t<std::is_arithmetic<T>::value>>::value[2]; #endif @@ -767,40 +1079,47 @@ template <typename T> constexpr const char format_descriptor< struct error_scope { PyObject *type, *value, *trace; error_scope() { PyErr_Fetch(&type, &value, &trace); } + error_scope(const error_scope &) = delete; + error_scope &operator=(const error_scope &) = delete; ~error_scope() { PyErr_Restore(type, value, trace); } }; /// Dummy destructor wrapper that can be used to expose classes with a private destructor -struct nodelete { template <typename T> void operator()(T*) { } }; +struct nodelete { + template <typename T> + void operator()(T *) {} +}; PYBIND11_NAMESPACE_BEGIN(detail) template <typename... Args> struct overload_cast_impl { - constexpr overload_cast_impl() {}; // NOLINT(modernize-use-equals-default): MSVC 2015 needs this - template <typename Return> - constexpr auto operator()(Return (*pf)(Args...)) const noexcept - -> decltype(pf) { return pf; } + constexpr auto operator()(Return (*pf)(Args...)) const noexcept -> decltype(pf) { + return pf; + } template <typename Return, typename Class> constexpr auto operator()(Return (Class::*pmf)(Args...), std::false_type = {}) const noexcept - -> decltype(pmf) { return pmf; } + -> decltype(pmf) { + return pmf; + } template <typename Return, typename Class> constexpr auto operator()(Return (Class::*pmf)(Args...) const, std::true_type) const noexcept - -> decltype(pmf) { return pmf; } + -> decltype(pmf) { + return pmf; + } }; PYBIND11_NAMESPACE_END(detail) // overload_cast requires variable templates: C++14 #if defined(PYBIND11_CPP14) -#define PYBIND11_OVERLOAD_CAST 1 +# define PYBIND11_OVERLOAD_CAST 1 /// Syntax sugar for resolving overloaded function pointers: /// - regular: static_cast<Return (Class::*)(Arg0, Arg1, Arg2)>(&Class::func) /// - sweet: overload_cast<Arg0, Arg1, Arg2>(&Class::func) template <typename... Args> -static constexpr detail::overload_cast_impl<Args...> overload_cast = {}; -// MSVC 2015 only accepts this particular initialization syntax for this variable template. +static constexpr detail::overload_cast_impl<Args...> overload_cast{}; #endif /// Const member function selector for overload_cast @@ -809,7 +1128,8 @@ static constexpr detail::overload_cast_impl<Args...> overload_cast = {}; static constexpr auto const_ = std::true_type{}; #if !defined(PYBIND11_CPP14) // no overload_cast: providing something that static_assert-fails: -template <typename... Args> struct overload_cast { +template <typename... Args> +struct overload_cast { static_assert(detail::deferred_t<std::false_type, Args...>::value, "pybind11::overload_cast<...> requires compiling in C++14 mode"); }; @@ -823,26 +1143,34 @@ PYBIND11_NAMESPACE_BEGIN(detail) template <typename T> class any_container { std::vector<T> v; + public: any_container() = default; // Can construct from a pair of iterators template <typename It, typename = enable_if_t<is_input_iterator<It>::value>> - any_container(It first, It last) : v(first, last) { } - - // Implicit conversion constructor from any arbitrary container type with values convertible to T - template <typename Container, typename = enable_if_t<std::is_convertible<decltype(*std::begin(std::declval<const Container &>())), T>::value>> - any_container(const Container &c) : any_container(std::begin(c), std::end(c)) { } - - // initializer_list's aren't deducible, so don't get matched by the above template; we need this - // to explicitly allow implicit conversion from one: + any_container(It first, It last) : v(first, last) {} + + // Implicit conversion constructor from any arbitrary container type + // with values convertible to T + template <typename Container, + typename = enable_if_t< + std::is_convertible<decltype(*std::begin(std::declval<const Container &>())), + T>::value>> + // NOLINTNEXTLINE(google-explicit-constructor) + any_container(const Container &c) : any_container(std::begin(c), std::end(c)) {} + + // initializer_list's aren't deducible, so don't get matched by the above template; + // we need this to explicitly allow implicit conversion from one: template <typename TIn, typename = enable_if_t<std::is_convertible<TIn, T>::value>> - any_container(const std::initializer_list<TIn> &c) : any_container(c.begin(), c.end()) { } + any_container(const std::initializer_list<TIn> &c) : any_container(c.begin(), c.end()) {} // Avoid copying if given an rvalue vector of the correct type. - any_container(std::vector<T> &&v) : v(std::move(v)) { } + // NOLINTNEXTLINE(google-explicit-constructor) + any_container(std::vector<T> &&v) : v(std::move(v)) {} // Moves the vector out of an rvalue any_container + // NOLINTNEXTLINE(google-explicit-constructor) operator std::vector<T> &&() && { return std::move(v); } // Dereferencing obtains a reference to the underlying vector @@ -855,7 +1183,76 @@ public: }; // Forward-declaration; see detail/class.h -std::string get_fully_qualified_tp_name(PyTypeObject*); +std::string get_fully_qualified_tp_name(PyTypeObject *); + +template <typename T> +inline static std::shared_ptr<T> +try_get_shared_from_this(std::enable_shared_from_this<T> *holder_value_ptr) { +// Pre C++17, this code path exploits undefined behavior, but is known to work on many platforms. +// Use at your own risk! +// See also https://en.cppreference.com/w/cpp/memory/enable_shared_from_this, and in particular +// the `std::shared_ptr<Good> gp1 = not_so_good.getptr();` and `try`-`catch` parts of the example. +#if defined(__cpp_lib_enable_shared_from_this) && (!defined(_MSC_VER) || _MSC_VER >= 1912) + return holder_value_ptr->weak_from_this().lock(); +#else + try { + return holder_value_ptr->shared_from_this(); + } catch (const std::bad_weak_ptr &) { + return nullptr; + } +#endif +} + +// For silencing "unused" compiler warnings in special situations. +template <typename... Args> +#if defined(_MSC_VER) && _MSC_VER < 1920 // MSVC 2017 +constexpr +#endif + inline void + silence_unused_warnings(Args &&...) { +} + +// MSVC warning C4100: Unreferenced formal parameter +#if defined(_MSC_VER) && _MSC_VER <= 1916 +# define PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(...) \ + detail::silence_unused_warnings(__VA_ARGS__) +#else +# define PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(...) +#endif + +// GCC -Wunused-but-set-parameter All GCC versions (as of July 2021). +#if defined(__GNUG__) && !defined(__clang__) && !defined(__INTEL_COMPILER) +# define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) \ + detail::silence_unused_warnings(__VA_ARGS__) +#else +# define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) +#endif + +#if defined(__clang__) \ + && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ + available. */ \ + || (__clang_major__ >= 7 \ + && __clang_major__ <= 12) /* Clang 3, 5, 13, 14, 15 do not generate the warning. */ \ + ) +# define PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +// Example: +// tests/test_kwargs_and_defaults.cpp:46:68: error: local variable 'args' will be copied despite +// being returned by name [-Werror,-Wreturn-std-move] +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// test_kwargs_and_defaults.cpp:46:68: note: call 'std::move' explicitly to avoid copying +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// std::move(args) +#endif + +// Pybind offers detailed error messages by default for all builts that are debug (through the +// negation of NDEBUG). This can also be manually enabled by users, for any builds, through +// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) +# define PYBIND11_DETAILED_ERROR_MESSAGES +#endif PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index 92720cd5..635614b0 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -15,24 +15,26 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) #if !defined(_MSC_VER) -# define PYBIND11_DESCR_CONSTEXPR static constexpr +# define PYBIND11_DESCR_CONSTEXPR static constexpr #else -# define PYBIND11_DESCR_CONSTEXPR const +# define PYBIND11_DESCR_CONSTEXPR const #endif /* Concatenate type signatures at compile time */ template <size_t N, typename... Ts> struct descr { - char text[N + 1]; + char text[N + 1]{'\0'}; - constexpr descr() : text{'\0'} { } - constexpr descr(char const (&s)[N+1]) : descr(s, make_index_sequence<N>()) { } + constexpr descr() = default; + // NOLINTNEXTLINE(google-explicit-constructor) + constexpr descr(char const (&s)[N + 1]) : descr(s, make_index_sequence<N>()) {} template <size_t... Is> - constexpr descr(char const (&s)[N+1], index_sequence<Is...>) : text{s[Is]..., '\0'} { } + constexpr descr(char const (&s)[N + 1], index_sequence<Is...>) : text{s[Is]..., '\0'} {} template <typename... Chars> - constexpr descr(char c, Chars... cs) : text{c, static_cast<char>(cs)..., '\0'} { } + // NOLINTNEXTLINE(google-explicit-constructor) + constexpr descr(char c, Chars... cs) : text{c, static_cast<char>(cs)..., '\0'} {} static constexpr std::array<const std::type_info *, sizeof...(Ts) + 1> types() { return {{&typeid(Ts)..., nullptr}}; @@ -40,60 +42,129 @@ struct descr { }; template <size_t N1, size_t N2, typename... Ts1, typename... Ts2, size_t... Is1, size_t... Is2> -constexpr descr<N1 + N2, Ts1..., Ts2...> plus_impl(const descr<N1, Ts1...> &a, const descr<N2, Ts2...> &b, - index_sequence<Is1...>, index_sequence<Is2...>) { +constexpr descr<N1 + N2, Ts1..., Ts2...> plus_impl(const descr<N1, Ts1...> &a, + const descr<N2, Ts2...> &b, + index_sequence<Is1...>, + index_sequence<Is2...>) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(b); return {a.text[Is1]..., b.text[Is2]...}; } template <size_t N1, size_t N2, typename... Ts1, typename... Ts2> -constexpr descr<N1 + N2, Ts1..., Ts2...> operator+(const descr<N1, Ts1...> &a, const descr<N2, Ts2...> &b) { +constexpr descr<N1 + N2, Ts1..., Ts2...> operator+(const descr<N1, Ts1...> &a, + const descr<N2, Ts2...> &b) { return plus_impl(a, b, make_index_sequence<N1>(), make_index_sequence<N2>()); } template <size_t N> -constexpr descr<N - 1> _(char const(&text)[N]) { return descr<N - 1>(text); } -constexpr descr<0> _(char const(&)[1]) { return {}; } +constexpr descr<N - 1> const_name(char const (&text)[N]) { + return descr<N - 1>(text); +} +constexpr descr<0> const_name(char const (&)[1]) { return {}; } -template <size_t Rem, size_t... Digits> struct int_to_str : int_to_str<Rem/10, Rem%10, Digits...> { }; -template <size_t...Digits> struct int_to_str<0, Digits...> { +template <size_t Rem, size_t... Digits> +struct int_to_str : int_to_str<Rem / 10, Rem % 10, Digits...> {}; +template <size_t... Digits> +struct int_to_str<0, Digits...> { + // WARNING: This only works with C++17 or higher. static constexpr auto digits = descr<sizeof...(Digits)>(('0' + Digits)...); }; // Ternary description (like std::conditional) template <bool B, size_t N1, size_t N2> -constexpr enable_if_t<B, descr<N1 - 1>> _(char const(&text1)[N1], char const(&)[N2]) { - return _(text1); +constexpr enable_if_t<B, descr<N1 - 1>> const_name(char const (&text1)[N1], char const (&)[N2]) { + return const_name(text1); } template <bool B, size_t N1, size_t N2> -constexpr enable_if_t<!B, descr<N2 - 1>> _(char const(&)[N1], char const(&text2)[N2]) { - return _(text2); +constexpr enable_if_t<!B, descr<N2 - 1>> const_name(char const (&)[N1], char const (&text2)[N2]) { + return const_name(text2); } template <bool B, typename T1, typename T2> -constexpr enable_if_t<B, T1> _(const T1 &d, const T2 &) { return d; } +constexpr enable_if_t<B, T1> const_name(const T1 &d, const T2 &) { + return d; +} template <bool B, typename T1, typename T2> -constexpr enable_if_t<!B, T2> _(const T1 &, const T2 &d) { return d; } +constexpr enable_if_t<!B, T2> const_name(const T1 &, const T2 &d) { + return d; +} -template <size_t Size> auto constexpr _() -> decltype(int_to_str<Size / 10, Size % 10>::digits) { +template <size_t Size> +auto constexpr const_name() -> remove_cv_t<decltype(int_to_str<Size / 10, Size % 10>::digits)> { return int_to_str<Size / 10, Size % 10>::digits; } -template <typename Type> constexpr descr<1, Type> _() { return {'%'}; } +template <typename Type> +constexpr descr<1, Type> const_name() { + return {'%'}; +} + +// If "_" is defined as a macro, py::detail::_ cannot be provided. +// It is therefore best to use py::detail::const_name universally. +// This block is for backward compatibility only. +// (The const_name code is repeated to avoid introducing a "_" #define ourselves.) +#ifndef _ +# define PYBIND11_DETAIL_UNDERSCORE_BACKWARD_COMPATIBILITY +template <size_t N> +constexpr descr<N - 1> _(char const (&text)[N]) { + return const_name<N>(text); +} +template <bool B, size_t N1, size_t N2> +constexpr enable_if_t<B, descr<N1 - 1>> _(char const (&text1)[N1], char const (&text2)[N2]) { + return const_name<B, N1, N2>(text1, text2); +} +template <bool B, size_t N1, size_t N2> +constexpr enable_if_t<!B, descr<N2 - 1>> _(char const (&text1)[N1], char const (&text2)[N2]) { + return const_name<B, N1, N2>(text1, text2); +} +template <bool B, typename T1, typename T2> +constexpr enable_if_t<B, T1> _(const T1 &d1, const T2 &d2) { + return const_name<B, T1, T2>(d1, d2); +} +template <bool B, typename T1, typename T2> +constexpr enable_if_t<!B, T2> _(const T1 &d1, const T2 &d2) { + return const_name<B, T1, T2>(d1, d2); +} + +template <size_t Size> +auto constexpr _() -> remove_cv_t<decltype(int_to_str<Size / 10, Size % 10>::digits)> { + return const_name<Size>(); +} +template <typename Type> +constexpr descr<1, Type> _() { + return const_name<Type>(); +} +#endif // #ifndef _ constexpr descr<0> concat() { return {}; } template <size_t N, typename... Ts> -constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) { return descr; } +constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) { + return descr; +} +#ifdef __cpp_fold_expressions +template <size_t N1, size_t N2, typename... Ts1, typename... Ts2> +constexpr descr<N1 + N2 + 2, Ts1..., Ts2...> operator,(const descr<N1, Ts1...> &a, + const descr<N2, Ts2...> &b) { + return a + const_name(", ") + b; +} + +template <size_t N, typename... Ts, typename... Args> +constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) { + return (d, ..., args); +} +#else template <size_t N, typename... Ts, typename... Args> constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) -> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) { - return d + _(", ") + concat(args...); + return d + const_name(", ") + concat(args...); } +#endif template <size_t N, typename... Ts> constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) { - return _("{") + descr + _("}"); + return const_name("{") + descr + const_name("}"); } PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 3ef78c11..e2117168 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -12,6 +12,9 @@ #include "class.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <> @@ -22,9 +25,10 @@ public: return true; } - template <typename> using cast_op_type = value_and_holder &; - operator value_and_holder &() { return *value; } - static constexpr auto name = _<value_and_holder>(); + template <typename> + using cast_op_type = value_and_holder &; + explicit operator value_and_holder &() { return *value; } + static constexpr auto name = const_name<value_and_holder>(); private: value_and_holder *value = nullptr; @@ -33,15 +37,21 @@ private: PYBIND11_NAMESPACE_BEGIN(initimpl) inline void no_nullptr(void *ptr) { - if (!ptr) throw type_error("pybind11::init(): factory function returned nullptr"); + if (!ptr) { + throw type_error("pybind11::init(): factory function returned nullptr"); + } } // Implementing functions for all forms of py::init<...> and py::init(...) -template <typename Class> using Cpp = typename Class::type; -template <typename Class> using Alias = typename Class::type_alias; -template <typename Class> using Holder = typename Class::holder_type; +template <typename Class> +using Cpp = typename Class::type; +template <typename Class> +using Alias = typename Class::type_alias; +template <typename Class> +using Holder = typename Class::holder_type; -template <typename Class> using is_alias_constructible = std::is_constructible<Alias<Class>, Cpp<Class> &&>; +template <typename Class> +using is_alias_constructible = std::is_constructible<Alias<Class>, Cpp<Class> &&>; // Takes a Cpp pointer and returns true if it actually is a polymorphic Alias instance. template <typename Class, enable_if_t<Class::has_alias, int> = 0> @@ -50,17 +60,27 @@ bool is_alias(Cpp<Class> *ptr) { } // Failing fallback version of the above for a no-alias class (always returns false) template <typename /*Class*/> -constexpr bool is_alias(void *) { return false; } +constexpr bool is_alias(void *) { + return false; +} // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall // back to brace aggregate initiailization so that for aggregate initialization can be used with // py::init, e.g. `py::init<int, int>` to initialize a `struct T { int a; int b; }`. For // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually // works, but will not do the expected thing when `T` has an `initializer_list<T>` constructor). -template <typename Class, typename... Args, detail::enable_if_t<std::is_constructible<Class, Args...>::value, int> = 0> -inline Class *construct_or_initialize(Args &&...args) { return new Class(std::forward<Args>(args)...); } -template <typename Class, typename... Args, detail::enable_if_t<!std::is_constructible<Class, Args...>::value, int> = 0> -inline Class *construct_or_initialize(Args &&...args) { return new Class{std::forward<Args>(args)...}; } +template <typename Class, + typename... Args, + detail::enable_if_t<std::is_constructible<Class, Args...>::value, int> = 0> +inline Class *construct_or_initialize(Args &&...args) { + return new Class(std::forward<Args>(args)...); +} +template <typename Class, + typename... Args, + detail::enable_if_t<!std::is_constructible<Class, Args...>::value, int> = 0> +inline Class *construct_or_initialize(Args &&...args) { + return new Class{std::forward<Args>(args)...}; +} // Attempts to constructs an alias using a `Alias(Cpp &&)` constructor. This allows types with // an alias to provide only a single Cpp factory function as long as the Alias can be @@ -69,12 +89,14 @@ inline Class *construct_or_initialize(Args &&...args) { return new Class{std::fo // inherit all the base class constructors. template <typename Class> void construct_alias_from_cpp(std::true_type /*is_alias_constructible*/, - value_and_holder &v_h, Cpp<Class> &&base) { + value_and_holder &v_h, + Cpp<Class> &&base) { v_h.value_ptr() = new Alias<Class>(std::move(base)); } template <typename Class> [[noreturn]] void construct_alias_from_cpp(std::false_type /*!is_alias_constructible*/, - value_and_holder &, Cpp<Class> &&) { + value_and_holder &, + Cpp<Class> &&) { throw type_error("pybind11::init(): unable to convert returned instance to required " "alias class: no `Alias<Class>(Class &&)` constructor available"); } @@ -84,8 +106,8 @@ template <typename Class> template <typename Class> void construct(...) { static_assert(!std::is_same<Class, Class>::value /* always false */, - "pybind11::init(): init function must return a compatible pointer, " - "holder, or value"); + "pybind11::init(): init function must return a compatible pointer, " + "holder, or value"); } // Pointer return v1: the factory function returns a class pointer for a registered class. @@ -94,6 +116,7 @@ void construct(...) { // construct an Alias from the returned base instance. template <typename Class> void construct(value_and_holder &v_h, Cpp<Class> *ptr, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); no_nullptr(ptr); if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) { // We're going to try to construct an alias by moving the cpp type. Whether or not @@ -105,7 +128,7 @@ void construct(value_and_holder &v_h, Cpp<Class> *ptr, bool need_alias) { // the holder and destruction happens when we leave the C++ scope, and the holder // class gets to handle the destruction however it likes. v_h.value_ptr() = ptr; - v_h.set_instance_registered(true); // To prevent init_instance from registering it + v_h.set_instance_registered(true); // To prevent init_instance from registering it v_h.type->init_instance(v_h.inst, nullptr); // Set up the holder Holder<Class> temp_holder(std::move(v_h.holder<Holder<Class>>())); // Steal the holder v_h.type->dealloc(v_h); // Destroys the moved-out holder remains, resets value ptr to null @@ -128,15 +151,18 @@ void construct(value_and_holder &v_h, Alias<Class> *alias_ptr, bool) { // Holder return: copy its pointer, and move or copy the returned holder into the new instance's // holder. This also handles types like std::shared_ptr<T> and std::unique_ptr<T> where T is a -// derived type (through those holder's implicit conversion from derived class holder constructors). +// derived type (through those holder's implicit conversion from derived class holder +// constructors). template <typename Class> void construct(value_and_holder &v_h, Holder<Class> holder, bool need_alias) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); auto *ptr = holder_helper<Holder<Class>>::get(holder); no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance - if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) + if (Class::has_alias && need_alias && !is_alias<Class>(ptr)) { throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " "is not an alias instance"); + } v_h.value_ptr() = ptr; v_h.type->init_instance(v_h.inst, &holder); @@ -148,12 +174,14 @@ void construct(value_and_holder &v_h, Holder<Class> holder, bool need_alias) { // need it, we simply move-construct the cpp value into a new instance. template <typename Class> void construct(value_and_holder &v_h, Cpp<Class> &&result, bool need_alias) { - static_assert(std::is_move_constructible<Cpp<Class>>::value, - "pybind11::init() return-by-value factory function requires a movable class"); - if (Class::has_alias && need_alias) + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); + static_assert(is_move_constructible<Cpp<Class>>::value, + "pybind11::init() return-by-value factory function requires a movable class"); + if (Class::has_alias && need_alias) { construct_alias_from_cpp<Class>(is_alias_constructible<Class>{}, v_h, std::move(result)); - else + } else { v_h.value_ptr() = new Cpp<Class>(std::move(result)); + } } // return-by-value version 2: returning a value of the alias type itself. We move-construct an @@ -161,7 +189,8 @@ void construct(value_and_holder &v_h, Cpp<Class> &&result, bool need_alias) { // cases where Alias initialization is always desired. template <typename Class> void construct(value_and_holder &v_h, Alias<Class> &&result, bool) { - static_assert(std::is_move_constructible<Alias<Class>>::value, + static_assert( + is_move_constructible<Alias<Class>>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias<Class>(std::move(result)); } @@ -170,48 +199,79 @@ void construct(value_and_holder &v_h, Alias<Class> &&result, bool) { template <typename... Args> struct constructor { template <typename Class, typename... Extra, enable_if_t<!Class::has_alias, int> = 0> - static void execute(Class &cl, const Extra&... extra) { - cl.def("__init__", [](value_and_holder &v_h, Args... args) { - v_h.value_ptr() = construct_or_initialize<Cpp<Class>>(std::forward<Args>(args)...); - }, is_new_style_constructor(), extra...); + static void execute(Class &cl, const Extra &...extra) { + cl.def( + "__init__", + [](value_and_holder &v_h, Args... args) { + v_h.value_ptr() = construct_or_initialize<Cpp<Class>>(std::forward<Args>(args)...); + }, + is_new_style_constructor(), + extra...); } - template <typename Class, typename... Extra, - enable_if_t<Class::has_alias && - std::is_constructible<Cpp<Class>, Args...>::value, int> = 0> - static void execute(Class &cl, const Extra&... extra) { - cl.def("__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(v_h.inst) == v_h.type->type) - v_h.value_ptr() = construct_or_initialize<Cpp<Class>>(std::forward<Args>(args)...); - else - v_h.value_ptr() = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...); - }, is_new_style_constructor(), extra...); + template < + typename Class, + typename... Extra, + enable_if_t<Class::has_alias && std::is_constructible<Cpp<Class>, Args...>::value, int> + = 0> + static void execute(Class &cl, const Extra &...extra) { + cl.def( + "__init__", + [](value_and_holder &v_h, Args... args) { + if (Py_TYPE(v_h.inst) == v_h.type->type) { + v_h.value_ptr() + = construct_or_initialize<Cpp<Class>>(std::forward<Args>(args)...); + } else { + v_h.value_ptr() + = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...); + } + }, + is_new_style_constructor(), + extra...); } - template <typename Class, typename... Extra, - enable_if_t<Class::has_alias && - !std::is_constructible<Cpp<Class>, Args...>::value, int> = 0> - static void execute(Class &cl, const Extra&... extra) { - cl.def("__init__", [](value_and_holder &v_h, Args... args) { - v_h.value_ptr() = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...); - }, is_new_style_constructor(), extra...); + template < + typename Class, + typename... Extra, + enable_if_t<Class::has_alias && !std::is_constructible<Cpp<Class>, Args...>::value, int> + = 0> + static void execute(Class &cl, const Extra &...extra) { + cl.def( + "__init__", + [](value_and_holder &v_h, Args... args) { + v_h.value_ptr() + = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...); + }, + is_new_style_constructor(), + extra...); } }; // Implementing class for py::init_alias<...>() -template <typename... Args> struct alias_constructor { - template <typename Class, typename... Extra, - enable_if_t<Class::has_alias && std::is_constructible<Alias<Class>, Args...>::value, int> = 0> - static void execute(Class &cl, const Extra&... extra) { - cl.def("__init__", [](value_and_holder &v_h, Args... args) { - v_h.value_ptr() = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...); - }, is_new_style_constructor(), extra...); +template <typename... Args> +struct alias_constructor { + template < + typename Class, + typename... Extra, + enable_if_t<Class::has_alias && std::is_constructible<Alias<Class>, Args...>::value, int> + = 0> + static void execute(Class &cl, const Extra &...extra) { + cl.def( + "__init__", + [](value_and_holder &v_h, Args... args) { + v_h.value_ptr() + = construct_or_initialize<Alias<Class>>(std::forward<Args>(args)...); + }, + is_new_style_constructor(), + extra...); } }; // Implementation class for py::init(Func) and py::init(Func, AliasFunc) -template <typename CFunc, typename AFunc = void_type (*)(), - typename = function_signature_t<CFunc>, typename = function_signature_t<AFunc>> +template <typename CFunc, + typename AFunc = void_type (*)(), + typename = function_signature_t<CFunc>, + typename = function_signature_t<AFunc>> struct factory; // Specialization for py::init(Func) @@ -219,7 +279,8 @@ template <typename Func, typename Return, typename... Args> struct factory<Func, void_type (*)(), Return(Args...)> { remove_reference_t<Func> class_factory; - factory(Func &&f) : class_factory(std::forward<Func>(f)) { } + // NOLINTNEXTLINE(google-explicit-constructor) + factory(Func &&f) : class_factory(std::forward<Func>(f)) {} // The given class either has no alias or has no separate alias factory; // this always constructs the class itself. If the class is registered with an alias @@ -228,22 +289,32 @@ struct factory<Func, void_type (*)(), Return(Args...)> { // instance, or the alias needs to be constructible from a `Class &&` argument. template <typename Class, typename... Extra> void execute(Class &cl, const Extra &...extra) && { - #if defined(PYBIND11_CPP14) - cl.def("__init__", [func = std::move(class_factory)] - #else +#if defined(PYBIND11_CPP14) + cl.def( + "__init__", + [func = std::move(class_factory)] +#else auto &func = class_factory; - cl.def("__init__", [func] - #endif - (value_and_holder &v_h, Args... args) { - construct<Class>(v_h, func(std::forward<Args>(args)...), - Py_TYPE(v_h.inst) != v_h.type->type); - }, is_new_style_constructor(), extra...); + cl.def( + "__init__", + [func] +#endif + (value_and_holder &v_h, Args... args) { + construct<Class>( + v_h, func(std::forward<Args>(args)...), Py_TYPE(v_h.inst) != v_h.type->type); + }, + is_new_style_constructor(), + extra...); } }; // Specialization for py::init(Func, AliasFunc) -template <typename CFunc, typename AFunc, - typename CReturn, typename... CArgs, typename AReturn, typename... AArgs> +template <typename CFunc, + typename AFunc, + typename CReturn, + typename... CArgs, + typename AReturn, + typename... AArgs> struct factory<CFunc, AFunc, CReturn(CArgs...), AReturn(AArgs...)> { static_assert(sizeof...(CArgs) == sizeof...(AArgs), "pybind11::init(class_factory, alias_factory): class and alias factories " @@ -256,29 +327,37 @@ struct factory<CFunc, AFunc, CReturn(CArgs...), AReturn(AArgs...)> { remove_reference_t<AFunc> alias_factory; factory(CFunc &&c, AFunc &&a) - : class_factory(std::forward<CFunc>(c)), alias_factory(std::forward<AFunc>(a)) { } + : class_factory(std::forward<CFunc>(c)), alias_factory(std::forward<AFunc>(a)) {} // The class factory is called when the `self` type passed to `__init__` is the direct // class (i.e. not inherited), the alias factory when `self` is a Python-side subtype. template <typename Class, typename... Extra> - void execute(Class &cl, const Extra&... extra) && { - static_assert(Class::has_alias, "The two-argument version of `py::init()` can " - "only be used if the class has an alias"); - #if defined(PYBIND11_CPP14) - cl.def("__init__", [class_func = std::move(class_factory), alias_func = std::move(alias_factory)] - #else + void execute(Class &cl, const Extra &...extra) && { + static_assert(Class::has_alias, + "The two-argument version of `py::init()` can " + "only be used if the class has an alias"); +#if defined(PYBIND11_CPP14) + cl.def( + "__init__", + [class_func = std::move(class_factory), alias_func = std::move(alias_factory)] +#else auto &class_func = class_factory; auto &alias_func = alias_factory; - cl.def("__init__", [class_func, alias_func] - #endif - (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(v_h.inst) == v_h.type->type) - // If the instance type equals the registered type we don't have inheritance, so - // don't need the alias and can construct using the class function: - construct<Class>(v_h, class_func(std::forward<CArgs>(args)...), false); - else - construct<Class>(v_h, alias_func(std::forward<CArgs>(args)...), true); - }, is_new_style_constructor(), extra...); + cl.def( + "__init__", + [class_func, alias_func] +#endif + (value_and_holder &v_h, CArgs... args) { + if (Py_TYPE(v_h.inst) == v_h.type->type) { + // If the instance type equals the registered type we don't have inheritance, + // so don't need the alias and can construct using the class function: + construct<Class>(v_h, class_func(std::forward<CArgs>(args)...), false); + } else { + construct<Class>(v_h, alias_func(std::forward<CArgs>(args)...), true); + } + }, + is_new_style_constructor(), + extra...); } }; @@ -289,20 +368,34 @@ void setstate(value_and_holder &v_h, T &&result, bool need_alias) { } /// Set both the C++ and Python states -template <typename Class, typename T, typename O, +template <typename Class, + typename T, + typename O, enable_if_t<std::is_convertible<O, handle>::value, int> = 0> void setstate(value_and_holder &v_h, std::pair<T, O> &&result, bool need_alias) { construct<Class>(v_h, std::move(result.first), need_alias); - setattr((PyObject *) v_h.inst, "__dict__", result.second); + auto d = handle(result.second); + if (PyDict_Check(d.ptr()) && PyDict_Size(d.ptr()) == 0) { + // Skipping setattr below, to not force use of py::dynamic_attr() for Class unnecessarily. + // See PR #2972 for details. + return; + } + setattr((PyObject *) v_h.inst, "__dict__", d); } /// Implementation for py::pickle(GetState, SetState) -template <typename Get, typename Set, - typename = function_signature_t<Get>, typename = function_signature_t<Set>> +template <typename Get, + typename Set, + typename = function_signature_t<Get>, + typename = function_signature_t<Set>> struct pickle_factory; -template <typename Get, typename Set, - typename RetState, typename Self, typename NewInstance, typename ArgState> +template <typename Get, + typename Set, + typename RetState, + typename Self, + typename NewInstance, + typename ArgState> struct pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)> { static_assert(std::is_same<intrinsic_t<RetState>, intrinsic_t<ArgState>>::value, "The type returned by `__getstate__` must be the same " @@ -311,26 +404,31 @@ struct pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)> { remove_reference_t<Get> get; remove_reference_t<Set> set; - pickle_factory(Get get, Set set) - : get(std::forward<Get>(get)), set(std::forward<Set>(set)) { } + pickle_factory(Get get, Set set) : get(std::forward<Get>(get)), set(std::forward<Set>(set)) {} template <typename Class, typename... Extra> void execute(Class &cl, const Extra &...extra) && { cl.def("__getstate__", std::move(get)); #if defined(PYBIND11_CPP14) - cl.def("__setstate__", [func = std::move(set)] + cl.def( + "__setstate__", + [func = std::move(set)] #else auto &func = set; - cl.def("__setstate__", [func] + cl.def( + "__setstate__", + [func] #endif - (value_and_holder &v_h, ArgState state) { - setstate<Class>(v_h, func(std::forward<ArgState>(state)), - Py_TYPE(v_h.inst) != v_h.type->type); - }, is_new_style_constructor(), extra...); + (value_and_holder &v_h, ArgState state) { + setstate<Class>( + v_h, func(std::forward<ArgState>(state)), Py_TYPE(v_h.inst) != v_h.type->type); + }, + is_new_style_constructor(), + extra...); } }; PYBIND11_NAMESPACE_END(initimpl) PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(pybind11) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 75fcd3c2..aaa7f868 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -9,10 +9,51 @@ #pragma once +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + #include "../pytypes.h" +#include <exception> + +/// Tracks the `internals` and `type_info` ABI version independent of the main library version. +/// +/// Some portions of the code use an ABI that is conditional depending on this +/// version number. That allows ABI-breaking changes to be "pre-implemented". +/// Once the default version number is incremented, the conditional logic that +/// no longer applies can be removed. Additionally, users that need not +/// maintain ABI compatibility can increase the version number in order to take +/// advantage of any functionality/efficiency improvements that depend on the +/// newer ABI. +/// +/// WARNING: If you choose to manually increase the ABI version, note that +/// pybind11 may not be tested as thoroughly with a non-default ABI version, and +/// further ABI-incompatible changes may be made before the ABI is officially +/// changed to the new version. +#ifndef PYBIND11_INTERNALS_VERSION +# if PY_VERSION_HEX >= 0x030C0000 +// Version bump for Python 3.12+, before first 3.12 beta release. +# define PYBIND11_INTERNALS_VERSION 5 +# else +# define PYBIND11_INTERNALS_VERSION 4 +# endif +#endif + +// This requirement is mainly to reduce the support burden (see PR #4570). +static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, + "pybind11 ABI version 5 is the minimum for Python 3.12+"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +using ExceptionTranslator = void (*)(std::exception_ptr); + PYBIND11_NAMESPACE_BEGIN(detail) + +constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule"; + // Forward declarations inline PyTypeObject *make_static_property_type(); inline PyTypeObject *make_default_metaclass(); @@ -21,30 +62,59 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new // Thread Specific Storage (TSS) API. #if PY_VERSION_HEX >= 0x03070000 -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr -# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key)) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) -# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) -# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) +// Avoid unnecessary allocation of `Py_tss_t`, since we cannot use +// `Py_LIMITED_API` anyway. +# if PYBIND11_INTERNALS_VERSION > 4 +# define PYBIND11_TLS_KEY_REF Py_tss_t & +# if defined(__GNUC__) && !defined(__INTEL_COMPILER) +// Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer +// for every field. +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("GCC diagnostic push") /**/ \ + _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var \ + = Py_tss_NEEDS_INIT; \ + _Pragma("GCC diagnostic pop") +# else +# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; +# endif +# define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) +# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) +# else +# define PYBIND11_TLS_KEY_REF Py_tss_t * +# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t *var = nullptr; +# define PYBIND11_TLS_KEY_CREATE(var) \ + (((var) = PyThread_tss_alloc()) != nullptr && (PyThread_tss_create((var)) == 0)) +# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key)) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) +# endif #else - // Usually an int but a long on Cygwin64 with Python 3.x -# define PYBIND11_TLS_KEY_INIT(var) decltype(PyThread_create_key()) var = 0 +// Usually an int but a long on Cygwin64 with Python 3.x +# define PYBIND11_TLS_KEY_REF decltype(PyThread_create_key()) +# define PYBIND11_TLS_KEY_INIT(var) PYBIND11_TLS_KEY_REF var = 0; +# define PYBIND11_TLS_KEY_CREATE(var) (((var) = PyThread_create_key()) != -1) # define PYBIND11_TLS_GET_VALUE(key) PyThread_get_key_value((key)) -# if PY_MAJOR_VERSION < 3 -# define PYBIND11_TLS_DELETE_VALUE(key) \ - PyThread_delete_key_value(key) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) \ - do { \ - PyThread_delete_key_value((key)); \ - PyThread_set_key_value((key), (value)); \ - } while (false) +# if defined(PYPY_VERSION) +// On CPython < 3.4 and on PyPy, `PyThread_set_key_value` strangely does not set +// the value if it has already been set. Instead, it must first be deleted and +// then set again. +inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { + PyThread_delete_key_value(key); + PyThread_set_key_value(key, value); +} +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_delete_key_value(key) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) \ + ::pybind11::detail::tls_replace_value((key), (value)) # else -# define PYBIND11_TLS_DELETE_VALUE(key) \ - PyThread_set_key_value((key), nullptr) -# define PYBIND11_TLS_REPLACE_VALUE(key, value) \ - PyThread_set_key_value((key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_set_key_value((key), nullptr) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_set_key_value((key), (value)) # endif -# define PYBIND11_TLS_FREE(key) (void)key +# define PYBIND11_TLS_FREE(key) (void) key #endif // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly @@ -53,7 +123,8 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash<std::type_index>; using type_equal_to = std::equal_to<std::type_index>; @@ -66,8 +137,9 @@ struct type_hash { size_t operator()(const std::type_index &t) const { size_t hash = 5381; const char *ptr = t.name(); - while (auto c = static_cast<unsigned char>(*ptr++)) + while (auto c = static_cast<unsigned char>(*ptr++)) { hash = (hash * 33) ^ c; + } return hash; } }; @@ -83,9 +155,9 @@ template <typename value_type> using type_map = std::unordered_map<std::type_index, value_type, type_hash, type_equal_to>; struct override_hash { - inline size_t operator()(const std::pair<const PyObject *, const char *>& v) const { + inline size_t operator()(const std::pair<const PyObject *, const char *> &v) const { size_t value = std::hash<const void *>()(v.first); - value ^= std::hash<const void *>()(v.second) + 0x9e3779b9 + (value<<6) + (value>>2); + value ^= std::hash<const void *>()(v.second) + 0x9e3779b9 + (value << 6) + (value >> 2); return value; } }; @@ -94,30 +166,56 @@ struct override_hash { /// Whenever binary incompatible changes are made to this structure, /// `PYBIND11_INTERNALS_VERSION` must be incremented. struct internals { - type_map<type_info *> registered_types_cpp; // std::type_index -> pybind11's type information - std::unordered_map<PyTypeObject *, std::vector<type_info *>> registered_types_py; // PyTypeObject* -> base type_info(s) - std::unordered_multimap<const void *, instance*> registered_instances; // void * -> instance* - std::unordered_set<std::pair<const PyObject *, const char *>, override_hash> inactive_override_cache; + // std::type_index -> pybind11's type information + type_map<type_info *> registered_types_cpp; + // PyTypeObject* -> base type_info(s) + std::unordered_map<PyTypeObject *, std::vector<type_info *>> registered_types_py; + std::unordered_multimap<const void *, instance *> registered_instances; // void * -> instance* + std::unordered_set<std::pair<const PyObject *, const char *>, override_hash> + inactive_override_cache; type_map<std::vector<bool (*)(PyObject *, void *&)>> direct_conversions; std::unordered_map<const PyObject *, std::vector<PyObject *>> patients; - std::forward_list<void (*) (std::exception_ptr)> registered_exception_translators; - std::unordered_map<std::string, void *> shared_data; // Custom data to be shared across extensions - std::vector<PyObject *> loader_patient_stack; // Used by `loader_life_support` - std::forward_list<std::string> static_strings; // Stores the std::strings backing detail::c_str() + std::forward_list<ExceptionTranslator> registered_exception_translators; + std::unordered_map<std::string, void *> shared_data; // Custom data to be shared across + // extensions +#if PYBIND11_INTERNALS_VERSION == 4 + std::vector<PyObject *> unused_loader_patient_stack_remove_at_v5; +#endif + std::forward_list<std::string> static_strings; // Stores the std::strings backing + // detail::c_str() PyTypeObject *static_property_type; PyTypeObject *default_metaclass; PyObject *instance_base; #if defined(WITH_THREAD) - PYBIND11_TLS_KEY_INIT(tstate); + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: + PYBIND11_TLS_KEY_INIT(tstate) +# if PYBIND11_INTERNALS_VERSION > 4 + PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) +# endif // PYBIND11_INTERNALS_VERSION > 4 + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; + +# if PYBIND11_INTERNALS_VERSION > 4 + // Note that we have to use a std::string to allocate memory to ensure a unique address + // We want unique addresses since we use pointer equality to compare function records + std::string function_record_capsule_name = internals_function_record_capsule_name; +# endif + + internals() = default; + internals(const internals &other) = delete; + internals &operator=(const internals &other) = delete; ~internals() { +# if PYBIND11_INTERNALS_VERSION > 4 + PYBIND11_TLS_FREE(loader_life_support_tls_key); +# endif // PYBIND11_INTERNALS_VERSION > 4 + // This destructor is called *after* Py_Finalize() in finalize_interpreter(). - // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is called. - // PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does nothing. - // PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. - // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). Neither - // of those have anything to do with CPython internals. - // PyMem_RawFree *requires* that the `tstate` be allocated with the CPython allocator. + // That *SHOULD BE* fine. The following details what happens when PyThread_tss_free is + // called. PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does + // nothing. PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. + // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). + // Neither of those have anything to do with CPython internals. PyMem_RawFree *requires* + // that the `tstate` be allocated with the CPython allocator. PYBIND11_TLS_FREE(tstate); } #endif @@ -132,14 +230,16 @@ struct type_info { void *(*operator_new)(size_t); void (*init_instance)(instance *, const void *); void (*dealloc)(value_and_holder &v_h); - std::vector<PyObject *(*)(PyObject *, PyTypeObject *)> implicit_conversions; - std::vector<std::pair<const std::type_info *, void *(*)(void *)>> implicit_casts; + std::vector<PyObject *(*) (PyObject *, PyTypeObject *)> implicit_conversions; + std::vector<std::pair<const std::type_info *, void *(*) (void *)>> implicit_casts; std::vector<bool (*)(PyObject *, void *&)> *direct_conversions; buffer_info *(*get_buffer)(PyObject *, void *) = nullptr; void *get_buffer_data = nullptr; void *(*module_local_load)(PyObject *, const type_info *) = nullptr; /* A simple type never occurs as a (direct or indirect) parent - * of a class that makes use of multiple inheritance */ + * of a class that makes use of multiple inheritance. + * A type can be simple even if it has non-simple ancestors as long as it has no descendants. + */ bool simple_type : 1; /* True if there is no multiple inheritance in this type's inheritance tree */ bool simple_ancestors : 1; @@ -149,72 +249,73 @@ struct type_info { bool module_local : 1; }; -/// Tracks the `internals` and `type_info` ABI version independent of the main library version -#define PYBIND11_INTERNALS_VERSION 4 - /// On MSVC, debug and release builds are not ABI-compatible! #if defined(_MSC_VER) && defined(_DEBUG) -# define PYBIND11_BUILD_TYPE "_debug" +# define PYBIND11_BUILD_TYPE "_debug" #else -# define PYBIND11_BUILD_TYPE "" +# define PYBIND11_BUILD_TYPE "" #endif /// Let's assume that different compilers are ABI-incompatible. /// A user can manually set this string if they know their /// compiler is compatible. #ifndef PYBIND11_COMPILER_TYPE -# if defined(_MSC_VER) -# define PYBIND11_COMPILER_TYPE "_msvc" -# elif defined(__INTEL_COMPILER) -# define PYBIND11_COMPILER_TYPE "_icc" -# elif defined(__clang__) -# define PYBIND11_COMPILER_TYPE "_clang" -# elif defined(__PGI) -# define PYBIND11_COMPILER_TYPE "_pgi" -# elif defined(__MINGW32__) -# define PYBIND11_COMPILER_TYPE "_mingw" -# elif defined(__CYGWIN__) -# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" -# elif defined(__GNUC__) -# define PYBIND11_COMPILER_TYPE "_gcc" -# else -# define PYBIND11_COMPILER_TYPE "_unknown" -# endif +# if defined(_MSC_VER) +# define PYBIND11_COMPILER_TYPE "_msvc" +# elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_TYPE "_icc" +# elif defined(__clang__) +# define PYBIND11_COMPILER_TYPE "_clang" +# elif defined(__PGI) +# define PYBIND11_COMPILER_TYPE "_pgi" +# elif defined(__MINGW32__) +# define PYBIND11_COMPILER_TYPE "_mingw" +# elif defined(__CYGWIN__) +# define PYBIND11_COMPILER_TYPE "_gcc_cygwin" +# elif defined(__GNUC__) +# define PYBIND11_COMPILER_TYPE "_gcc" +# else +# define PYBIND11_COMPILER_TYPE "_unknown" +# endif #endif /// Also standard libs #ifndef PYBIND11_STDLIB -# if defined(_LIBCPP_VERSION) -# define PYBIND11_STDLIB "_libcpp" -# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) -# define PYBIND11_STDLIB "_libstdcpp" -# else -# define PYBIND11_STDLIB "" -# endif +# if defined(_LIBCPP_VERSION) +# define PYBIND11_STDLIB "_libcpp" +# elif defined(__GLIBCXX__) || defined(__GLIBCPP__) +# define PYBIND11_STDLIB "_libstdcpp" +# else +# define PYBIND11_STDLIB "" +# endif #endif /// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. #ifndef PYBIND11_BUILD_ABI -# if defined(__GXX_ABI_VERSION) -# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) -# else -# define PYBIND11_BUILD_ABI "" -# endif +# if defined(__GXX_ABI_VERSION) +# define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) +# else +# define PYBIND11_BUILD_ABI "" +# endif #endif #ifndef PYBIND11_INTERNALS_KIND -# if defined(WITH_THREAD) -# define PYBIND11_INTERNALS_KIND "" -# else -# define PYBIND11_INTERNALS_KIND "_without_thread" -# endif +# if defined(WITH_THREAD) +# define PYBIND11_INTERNALS_KIND "" +# else +# define PYBIND11_INTERNALS_KIND "_without_thread" +# endif #endif -#define PYBIND11_INTERNALS_ID "__pybind11_internals_v" \ - PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" +#define PYBIND11_INTERNALS_ID \ + "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ + PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ + PYBIND11_BUILD_TYPE "__" -#define PYBIND11_MODULE_LOCAL_ID "__pybind11_module_local_v" \ - PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI PYBIND11_BUILD_TYPE "__" +#define PYBIND11_MODULE_LOCAL_ID \ + "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ + PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI \ + PYBIND11_BUILD_TYPE "__" /// Each module locally stores a pointer to the `internals` data. The data /// itself is shared among modules with the same `PYBIND11_INTERNALS_ID`. @@ -223,21 +324,93 @@ inline internals **&get_internals_pp() { return internals_pp; } +// forward decl +inline void translate_exception(std::exception_ptr); + +template <class T, + enable_if_t<std::is_same<std::nested_exception, remove_cvref_t<T>>::value, int> = 0> +bool handle_nested_exception(const T &exc, const std::exception_ptr &p) { + std::exception_ptr nested = exc.nested_ptr(); + if (nested != nullptr && nested != p) { + translate_exception(nested); + return true; + } + return false; +} + +template <class T, + enable_if_t<!std::is_same<std::nested_exception, remove_cvref_t<T>>::value, int> = 0> +bool handle_nested_exception(const T &exc, const std::exception_ptr &p) { + if (const auto *nep = dynamic_cast<const std::nested_exception *>(std::addressof(exc))) { + return handle_nested_exception(*nep, p); + } + return false; +} + +inline bool raise_err(PyObject *exc_type, const char *msg) { + if (PyErr_Occurred()) { + raise_from(exc_type, msg); + return true; + } + PyErr_SetString(exc_type, msg); + return false; +} + inline void translate_exception(std::exception_ptr p) { + if (!p) { + return; + } try { - if (p) std::rethrow_exception(p); - } catch (error_already_set &e) { e.restore(); return; - } catch (const builtin_exception &e) { e.set_error(); return; - } catch (const std::bad_alloc &e) { PyErr_SetString(PyExc_MemoryError, e.what()); return; - } catch (const std::domain_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::invalid_argument &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::length_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::out_of_range &e) { PyErr_SetString(PyExc_IndexError, e.what()); return; - } catch (const std::range_error &e) { PyErr_SetString(PyExc_ValueError, e.what()); return; - } catch (const std::overflow_error &e) { PyErr_SetString(PyExc_OverflowError, e.what()); return; - } catch (const std::exception &e) { PyErr_SetString(PyExc_RuntimeError, e.what()); return; + std::rethrow_exception(p); + } catch (error_already_set &e) { + handle_nested_exception(e, p); + e.restore(); + return; + } catch (const builtin_exception &e) { + // Could not use template since it's an abstract class. + if (const auto *nep = dynamic_cast<const std::nested_exception *>(std::addressof(e))) { + handle_nested_exception(*nep, p); + } + e.set_error(); + return; + } catch (const std::bad_alloc &e) { + handle_nested_exception(e, p); + raise_err(PyExc_MemoryError, e.what()); + return; + } catch (const std::domain_error &e) { + handle_nested_exception(e, p); + raise_err(PyExc_ValueError, e.what()); + return; + } catch (const std::invalid_argument &e) { + handle_nested_exception(e, p); + raise_err(PyExc_ValueError, e.what()); + return; + } catch (const std::length_error &e) { + handle_nested_exception(e, p); + raise_err(PyExc_ValueError, e.what()); + return; + } catch (const std::out_of_range &e) { + handle_nested_exception(e, p); + raise_err(PyExc_IndexError, e.what()); + return; + } catch (const std::range_error &e) { + handle_nested_exception(e, p); + raise_err(PyExc_ValueError, e.what()); + return; + } catch (const std::overflow_error &e) { + handle_nested_exception(e, p); + raise_err(PyExc_OverflowError, e.what()); + return; + } catch (const std::exception &e) { + handle_nested_exception(e, p); + raise_err(PyExc_RuntimeError, e.what()); + return; + } catch (const std::nested_exception &e) { + handle_nested_exception(e, p); + raise_err(PyExc_RuntimeError, "Caught an unknown nested exception!"); + return; } catch (...) { - PyErr_SetString(PyExc_RuntimeError, "Caught an unknown exception!"); + raise_err(PyExc_RuntimeError, "Caught an unknown exception!"); return; } } @@ -245,64 +418,115 @@ inline void translate_exception(std::exception_ptr p) { #if !defined(__GLIBCXX__) inline void translate_local_exception(std::exception_ptr p) { try { - if (p) std::rethrow_exception(p); - } catch (error_already_set &e) { e.restore(); return; - } catch (const builtin_exception &e) { e.set_error(); return; + if (p) { + std::rethrow_exception(p); + } + } catch (error_already_set &e) { + e.restore(); + return; + } catch (const builtin_exception &e) { + e.set_error(); + return; } } #endif +inline object get_python_state_dict() { + object state_dict; +#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow<object>(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + } + return state_dict; +} + +inline object get_internals_obj_from_state_dict(handle state_dict) { + return reinterpret_borrow<object>(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); +} + +inline internals **get_internals_pp_from_capsule(handle obj) { + void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); + if (raw_ptr == nullptr) { + raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + } + return static_cast<internals **>(raw_ptr); +} + /// Return a reference to the current `internals` data -PYBIND11_NOINLINE inline internals &get_internals() { +PYBIND11_NOINLINE internals &get_internals() { auto **&internals_pp = get_internals_pp(); - if (internals_pp && *internals_pp) + if (internals_pp && *internals_pp) { return **internals_pp; + } +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + gil_scoped_acquire gil; +# else // Ensure that the GIL is held since we will need to make Python calls. // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { - gil_scoped_acquire_local() : state (PyGILState_Ensure()) {} + gil_scoped_acquire_local() : state(PyGILState_Ensure()) {} + gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete; + gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete; ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; +# endif +#endif + error_scope err_scope; - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance<capsule>(builtins[id])) { - internals_pp = static_cast<internals **>(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our `error_already_set` + dict state_dict = get_python_state_dict(); + if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) { + internals_pp = get_internals_pp_from_capsule(internals_obj); + } + if (internals_pp && *internals_pp) { + // We loaded the internals through `state_dict`, which means that our `error_already_set` // and `builtin_exception` may be different local classes than the ones set up in the // initial exception translator, below, so add another for our local exception classes. // // libstdc++ doesn't require this (types there are identified only by name) + // libc++ with CPython doesn't require this (types are explicitly exported) + // libc++ with PyPy still need it, awaiting further investigation #if !defined(__GLIBCXX__) (*internals_pp)->registered_exception_translators.push_front(&translate_local_exception); #endif } else { - if (!internals_pp) internals_pp = new internals*(); + if (!internals_pp) { + internals_pp = new internals *(); + } auto *&internals_ptr = *internals_pp; internals_ptr = new internals(); #if defined(WITH_THREAD) - #if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); - #endif PyThreadState *tstate = PyThreadState_Get(); - #if PY_VERSION_HEX >= 0x03070000 - internals_ptr->tstate = PyThread_tss_alloc(); - if (!internals_ptr->tstate || PyThread_tss_create(internals_ptr->tstate)) - pybind11_fail("get_internals: could not successfully initialize the TSS key!"); - PyThread_tss_set(internals_ptr->tstate, tstate); - #else - internals_ptr->tstate = PyThread_create_key(); - if (internals_ptr->tstate == -1) - pybind11_fail("get_internals: could not successfully initialize the TLS key!"); - PyThread_set_key_value(internals_ptr->tstate, tstate); - #endif + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { + pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); + } + PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); + +# if PYBIND11_INTERNALS_VERSION > 4 + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { + pybind11_fail("get_internals: could not successfully initialize the " + "loader_life_support TSS key!"); + } +# endif internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); @@ -311,10 +535,60 @@ PYBIND11_NOINLINE inline internals &get_internals() { return **internals_pp; } -/// Works like `internals.registered_types_cpp`, but for module-local registered types: -inline type_map<type_info *> ®istered_local_types_cpp() { - static type_map<type_info *> locals{}; - return locals; +// the internals struct (above) is shared between all the modules. local_internals are only +// for a single module. Any changes made to internals may require an update to +// PYBIND11_INTERNALS_VERSION, breaking backwards compatibility. local_internals is, by design, +// restricted to a single module. Whether a module has local internals or not should not +// impact any other modules, because the only things accessing the local internals is the +// module that contains them. +struct local_internals { + type_map<type_info *> registered_types_cpp; + std::forward_list<ExceptionTranslator> registered_exception_translators; +#if defined(WITH_THREAD) && PYBIND11_INTERNALS_VERSION == 4 + + // For ABI compatibility, we can't store the loader_life_support TLS key in + // the `internals` struct directly. Instead, we store it in `shared_data` and + // cache a copy in `local_internals`. If we allocated a separate TLS key for + // each instance of `local_internals`, we could end up allocating hundreds of + // TLS keys if hundreds of different pybind11 modules are loaded (which is a + // plausible number). + PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) + + // Holds the shared TLS key for the loader_life_support stack. + struct shared_loader_life_support_data { + PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) + shared_loader_life_support_data() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) + if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { + pybind11_fail("local_internals: could not successfully initialize the " + "loader_life_support TLS key!"); + } + } + // We can't help but leak the TLS key, because Python never unloads extension modules. + }; + + local_internals() { + auto &internals = get_internals(); + // Get or create the `loader_life_support_stack_key`. + auto &ptr = internals.shared_data["_life_support"]; + if (!ptr) { + ptr = new shared_loader_life_support_data; + } + loader_life_support_tls_key + = static_cast<shared_loader_life_support_data *>(ptr)->loader_life_support_tls_key; + } +#endif // defined(WITH_THREAD) && PYBIND11_INTERNALS_VERSION == 4 +}; + +/// Works like `get_internals`, but for things which are locally registered. +inline local_internals &get_local_internals() { + // Current static can be created in the interpreter finalization routine. If the later will be + // destroyed in another static variable destructor, creation of this static there will cause + // static deinitialization fiasco. In order to avoid it we avoid destruction of the + // local_internals static. One can read more about the problem and current solution here: + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *locals = new local_internals(); + return *locals; } /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its @@ -328,19 +602,38 @@ const char *c_str(Args &&...args) { return strings.front().c_str(); } +inline const char *get_function_record_capsule_name() { +#if PYBIND11_INTERNALS_VERSION > 4 + return get_internals().function_record_capsule_name.c_str(); +#else + return nullptr; +#endif +} + +// Determine whether or not the following capsule contains a pybind11 function record. +// Note that we use `internals` to make sure that only ABI compatible records are touched. +// +// This check is currently used in two places: +// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++ +// - The sibling feature of cpp_function to allow overloads +inline bool is_function_record_capsule(const capsule &cap) { + // Pointer equality as we rely on internals() to ensure unique pointers + return cap.name() == get_function_record_capsule_name(); +} + PYBIND11_NAMESPACE_END(detail) /// Returns a named pointer that is shared among all extension modules (using the same /// pybind11 version) running in the current interpreter. Names starting with underscores /// are reserved for internal usage. Returns `nullptr` if no matching entry was found. -inline PYBIND11_NOINLINE void *get_shared_data(const std::string &name) { +PYBIND11_NOINLINE void *get_shared_data(const std::string &name) { auto &internals = detail::get_internals(); auto it = internals.shared_data.find(name); return it != internals.shared_data.end() ? it->second : nullptr; } /// Set the shared data that can be later recovered by `get_shared_data()`. -inline PYBIND11_NOINLINE void *set_shared_data(const std::string &name, void *data) { +PYBIND11_NOINLINE void *set_shared_data(const std::string &name, void *data) { detail::get_internals().shared_data[name] = data; return data; } @@ -348,7 +641,7 @@ inline PYBIND11_NOINLINE void *set_shared_data(const std::string &name, void *da /// Returns a typed reference to a shared data entry (by using `get_shared_data()`) if /// such entry exists. Otherwise, a new object of default-constructible type `T` is /// added to the shared data under the given name and a reference to it is returned. -template<typename T> +template <typename T> T &get_or_create_shared_data(const std::string &name) { auto &internals = detail::get_internals(); auto it = internals.shared_data.find(name); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h new file mode 100644 index 00000000..16387506 --- /dev/null +++ b/include/pybind11/detail/type_caster_base.h @@ -0,0 +1,1177 @@ +/* + pybind11/detail/type_caster_base.h (originally first part of pybind11/cast.h) + + Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch> + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../pytypes.h" +#include "common.h" +#include "descr.h" +#include "internals.h" +#include "typeid.h" + +#include <cstdint> +#include <iterator> +#include <new> +#include <string> +#include <type_traits> +#include <typeindex> +#include <typeinfo> +#include <unordered_map> +#include <utility> +#include <vector> + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +/// A life support system for temporary objects created by `type_caster::load()`. +/// Adding a patient will keep it alive up until the enclosing function returns. +class loader_life_support { +private: + loader_life_support *parent = nullptr; + std::unordered_set<PyObject *> keep_alive; + +#if defined(WITH_THREAD) + // Store stack pointer in thread-local storage. + static PYBIND11_TLS_KEY_REF get_stack_tls_key() { +# if PYBIND11_INTERNALS_VERSION == 4 + return get_local_internals().loader_life_support_tls_key; +# else + return get_internals().loader_life_support_tls_key; +# endif + } + static loader_life_support *get_stack_top() { + return static_cast<loader_life_support *>(PYBIND11_TLS_GET_VALUE(get_stack_tls_key())); + } + static void set_stack_top(loader_life_support *value) { + PYBIND11_TLS_REPLACE_VALUE(get_stack_tls_key(), value); + } +#else + // Use single global variable for stack. + static loader_life_support **get_stack_pp() { + static loader_life_support *global_stack = nullptr; + return global_stack; + } + static loader_life_support *get_stack_top() { return *get_stack_pp(); } + static void set_stack_top(loader_life_support *value) { *get_stack_pp() = value; } +#endif + +public: + /// A new patient frame is created when a function is entered + loader_life_support() : parent{get_stack_top()} { set_stack_top(this); } + + /// ... and destroyed after it returns + ~loader_life_support() { + if (get_stack_top() != this) { + pybind11_fail("loader_life_support: internal error"); + } + set_stack_top(parent); + for (auto *item : keep_alive) { + Py_DECREF(item); + } + } + + /// This can only be used inside a pybind11-bound function, either by `argument_loader` + /// at argument preparation time or by `py::cast()` at execution time. + PYBIND11_NOINLINE static void add_patient(handle h) { + loader_life_support *frame = get_stack_top(); + if (!frame) { + // NOTE: It would be nice to include the stack frames here, as this indicates + // use of pybind11::cast<> outside the normal call framework, finding such + // a location is challenging. Developers could consider printing out + // stack frame addresses here using something like __builtin_frame_address(0) + throw cast_error("When called outside a bound function, py::cast() cannot " + "do Python -> C++ conversions which require the creation " + "of temporary values"); + } + + if (frame->keep_alive.insert(h.ptr()).second) { + Py_INCREF(h.ptr()); + } + } +}; + +// Gets the cache entry for the given type, creating it if necessary. The return value is the pair +// returned by emplace, i.e. an iterator for the entry and a bool set to `true` if the entry was +// just created. +inline std::pair<decltype(internals::registered_types_py)::iterator, bool> +all_type_info_get_cache(PyTypeObject *type); + +// Populates a just-created cache entry. +PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector<type_info *> &bases) { + std::vector<PyTypeObject *> check; + for (handle parent : reinterpret_borrow<tuple>(t->tp_bases)) { + check.push_back((PyTypeObject *) parent.ptr()); + } + + auto const &type_dict = get_internals().registered_types_py; + for (size_t i = 0; i < check.size(); i++) { + auto *type = check[i]; + // Ignore Python2 old-style class super type: + if (!PyType_Check((PyObject *) type)) { + continue; + } + + // Check `type` in the current set of registered python types: + auto it = type_dict.find(type); + if (it != type_dict.end()) { + // We found a cache entry for it, so it's either pybind-registered or has pre-computed + // pybind bases, but we have to make sure we haven't already seen the type(s) before: + // we want to follow Python/virtual C++ rules that there should only be one instance of + // a common base. + for (auto *tinfo : it->second) { + // NB: Could use a second set here, rather than doing a linear search, but since + // having a large number of immediate pybind11-registered types seems fairly + // unlikely, that probably isn't worthwhile. + bool found = false; + for (auto *known : bases) { + if (known == tinfo) { + found = true; + break; + } + } + if (!found) { + bases.push_back(tinfo); + } + } + } else if (type->tp_bases) { + // It's some python type, so keep follow its bases classes to look for one or more + // registered types + if (i + 1 == check.size()) { + // When we're at the end, we can pop off the current element to avoid growing + // `check` when adding just one base (which is typical--i.e. when there is no + // multiple inheritance) + check.pop_back(); + i--; + } + for (handle parent : reinterpret_borrow<tuple>(type->tp_bases)) { + check.push_back((PyTypeObject *) parent.ptr()); + } + } + } +} + +/** + * Extracts vector of type_info pointers of pybind-registered roots of the given Python type. Will + * be just 1 pybind type for the Python type of a pybind-registered class, or for any Python-side + * derived class that uses single inheritance. Will contain as many types as required for a Python + * class that uses multiple inheritance to inherit (directly or indirectly) from multiple + * pybind-registered classes. Will be empty if neither the type nor any base classes are + * pybind-registered. + * + * The value is cached for the lifetime of the Python type. + */ +inline const std::vector<detail::type_info *> &all_type_info(PyTypeObject *type) { + auto ins = all_type_info_get_cache(type); + if (ins.second) { + // New cache entry: populate it + all_type_info_populate(type, ins.first->second); + } + + return ins.first->second; +} + +/** + * Gets a single pybind11 type info for a python type. Returns nullptr if neither the type nor any + * ancestors are pybind11-registered. Throws an exception if there are multiple bases--use + * `all_type_info` instead if you want to support multiple bases. + */ +PYBIND11_NOINLINE detail::type_info *get_type_info(PyTypeObject *type) { + const auto &bases = all_type_info(type); + if (bases.empty()) { + return nullptr; + } + if (bases.size() > 1) { + pybind11_fail( + "pybind11::detail::get_type_info: type has multiple pybind11-registered bases"); + } + return bases.front(); +} + +inline detail::type_info *get_local_type_info(const std::type_index &tp) { + auto &locals = get_local_internals().registered_types_cpp; + auto it = locals.find(tp); + if (it != locals.end()) { + return it->second; + } + return nullptr; +} + +inline detail::type_info *get_global_type_info(const std::type_index &tp) { + auto &types = get_internals().registered_types_cpp; + auto it = types.find(tp); + if (it != types.end()) { + return it->second; + } + return nullptr; +} + +/// Return the type info for a given C++ type; on lookup failure can either throw or return +/// nullptr. +PYBIND11_NOINLINE detail::type_info *get_type_info(const std::type_index &tp, + bool throw_if_missing = false) { + if (auto *ltype = get_local_type_info(tp)) { + return ltype; + } + if (auto *gtype = get_global_type_info(tp)) { + return gtype; + } + + if (throw_if_missing) { + std::string tname = tp.name(); + detail::clean_type_id(tname); + pybind11_fail("pybind11::detail::get_type_info: unable to find type info for \"" + + std::move(tname) + '"'); + } + return nullptr; +} + +PYBIND11_NOINLINE handle get_type_handle(const std::type_info &tp, bool throw_if_missing) { + detail::type_info *type_info = get_type_info(tp, throw_if_missing); + return handle(type_info ? ((PyObject *) type_info->type) : nullptr); +} + +// Searches the inheritance graph for a registered Python instance, using all_type_info(). +PYBIND11_NOINLINE handle find_registered_python_instance(void *src, + const detail::type_info *tinfo) { + auto it_instances = get_internals().registered_instances.equal_range(src); + for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { + for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { + if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { + return handle((PyObject *) it_i->second).inc_ref(); + } + } + } + return handle(); +} + +struct value_and_holder { + instance *inst = nullptr; + size_t index = 0u; + const detail::type_info *type = nullptr; + void **vh = nullptr; + + // Main constructor for a found value/holder: + value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} + + // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) + value_and_holder() = default; + + // Used for past-the-end iterator + explicit value_and_holder(size_t index) : index{index} {} + + template <typename V = void> + V *&value_ptr() const { + return reinterpret_cast<V *&>(vh[0]); + } + // True if this `value_and_holder` has a non-null value pointer + explicit operator bool() const { return value_ptr() != nullptr; } + + template <typename H> + H &holder() const { + return reinterpret_cast<H &>(vh[1]); + } + bool holder_constructed() const { + return inst->simple_layout + ? inst->simple_holder_constructed + : (inst->nonsimple.status[index] & instance::status_holder_constructed) != 0u; + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void set_holder_constructed(bool v = true) { + if (inst->simple_layout) { + inst->simple_holder_constructed = v; + } else if (v) { + inst->nonsimple.status[index] |= instance::status_holder_constructed; + } else { + inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_holder_constructed; + } + } + bool instance_registered() const { + return inst->simple_layout + ? inst->simple_instance_registered + : ((inst->nonsimple.status[index] & instance::status_instance_registered) != 0); + } + // NOLINTNEXTLINE(readability-make-member-function-const) + void set_instance_registered(bool v = true) { + if (inst->simple_layout) { + inst->simple_instance_registered = v; + } else if (v) { + inst->nonsimple.status[index] |= instance::status_instance_registered; + } else { + inst->nonsimple.status[index] &= (std::uint8_t) ~instance::status_instance_registered; + } + } +}; + +// Container for accessing and iterating over an instance's values/holders +struct values_and_holders { +private: + instance *inst; + using type_vec = std::vector<detail::type_info *>; + const type_vec &tinfo; + +public: + explicit values_and_holders(instance *inst) + : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} + + struct iterator { + private: + instance *inst = nullptr; + const type_vec *types = nullptr; + value_and_holder curr; + friend struct values_and_holders; + iterator(instance *inst, const type_vec *tinfo) + : inst{inst}, types{tinfo}, + curr(inst /* instance */, + types->empty() ? nullptr : (*types)[0] /* type info */, + 0, /* vpos: (non-simple types only): the first vptr comes first */ + 0 /* index */) {} + // Past-the-end iterator: + explicit iterator(size_t end) : curr(end) {} + + public: + bool operator==(const iterator &other) const { return curr.index == other.curr.index; } + bool operator!=(const iterator &other) const { return curr.index != other.curr.index; } + iterator &operator++() { + if (!inst->simple_layout) { + curr.vh += 1 + (*types)[curr.index]->holder_size_in_ptrs; + } + ++curr.index; + curr.type = curr.index < types->size() ? (*types)[curr.index] : nullptr; + return *this; + } + value_and_holder &operator*() { return curr; } + value_and_holder *operator->() { return &curr; } + }; + + iterator begin() { return iterator(inst, &tinfo); } + iterator end() { return iterator(tinfo.size()); } + + iterator find(const type_info *find_type) { + auto it = begin(), endit = end(); + while (it != endit && it->type != find_type) { + ++it; + } + return it; + } + + size_t size() { return tinfo.size(); } +}; + +/** + * Extracts C++ value and holder pointer references from an instance (which may contain multiple + * values/holders for python-side multiple inheritance) that match the given type. Throws an error + * if the given type (or ValueType, if omitted) is not a pybind11 base of the given instance. If + * `find_type` is omitted (or explicitly specified as nullptr) the first value/holder are returned, + * regardless of type (and the resulting .type will be nullptr). + * + * The returned object should be short-lived: in particular, it must not outlive the called-upon + * instance. + */ +PYBIND11_NOINLINE value_and_holder +instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, + bool throw_if_missing /*= true in common.h*/) { + // Optimize common case: + if (!find_type || Py_TYPE(this) == find_type->type) { + return value_and_holder(this, find_type, 0, 0); + } + + detail::values_and_holders vhs(this); + auto it = vhs.find(find_type); + if (it != vhs.end()) { + return *it; + } + + if (!throw_if_missing) { + return value_and_holder(); + } + +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + + get_fully_qualified_tp_name(find_type->type) + + "' is not a pybind11 base of the given `" + + get_fully_qualified_tp_name(Py_TYPE(this)) + "' instance"); +#else + pybind11_fail( + "pybind11::detail::instance::get_value_and_holder: " + "type is not a pybind11 base of the given instance " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for type details)"); +#endif +} + +PYBIND11_NOINLINE void instance::allocate_layout() { + const auto &tinfo = all_type_info(Py_TYPE(this)); + + const size_t n_types = tinfo.size(); + + if (n_types == 0) { + pybind11_fail( + "instance allocation failed: new instance has no pybind11-registered base types"); + } + + simple_layout + = n_types == 1 && tinfo.front()->holder_size_in_ptrs <= instance_simple_holder_in_ptrs(); + + // Simple path: no python-side multiple inheritance, and a small-enough holder + if (simple_layout) { + simple_value_holder[0] = nullptr; + simple_holder_constructed = false; + simple_instance_registered = false; + } else { // multiple base types or a too-large holder + // Allocate space to hold: [v1*][h1][v2*][h2]...[bb...] where [vN*] is a value pointer, + // [hN] is the (uninitialized) holder instance for value N, and [bb...] is a set of bool + // values that tracks whether each associated holder has been initialized. Each [block] is + // padded, if necessary, to an integer multiple of sizeof(void *). + size_t space = 0; + for (auto *t : tinfo) { + space += 1; // value pointer + space += t->holder_size_in_ptrs; // holder instance + } + size_t flags_at = space; + space += size_in_ptrs(n_types); // status bytes (holder_constructed and + // instance_registered) + + // Allocate space for flags, values, and holders, and initialize it to 0 (flags and values, + // in particular, need to be 0). Use Python's memory allocation + // functions: Python is using pymalloc, which is designed to be + // efficient for small allocations like the one we're doing here; + // for larger allocations they are just wrappers around malloc. + // TODO: is this still true for pure Python 3.6? + nonsimple.values_and_holders = (void **) PyMem_Calloc(space, sizeof(void *)); + if (!nonsimple.values_and_holders) { + throw std::bad_alloc(); + } + nonsimple.status + = reinterpret_cast<std::uint8_t *>(&nonsimple.values_and_holders[flags_at]); + } + owned = true; +} + +// NOLINTNEXTLINE(readability-make-member-function-const) +PYBIND11_NOINLINE void instance::deallocate_layout() { + if (!simple_layout) { + PyMem_Free(nonsimple.values_and_holders); + } +} + +PYBIND11_NOINLINE bool isinstance_generic(handle obj, const std::type_info &tp) { + handle type = detail::get_type_handle(tp, false); + if (!type) { + return false; + } + return isinstance(obj, type); +} + +PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_info *type) { + auto &instances = get_internals().registered_instances; + auto range = instances.equal_range(ptr); + for (auto it = range.first; it != range.second; ++it) { + for (const auto &vh : values_and_holders(it->second)) { + if (vh.type == type) { + return handle((PyObject *) it->second); + } + } + } + return handle(); +} + +inline PyThreadState *get_thread_state_unchecked() { +#if defined(PYPY_VERSION) + return PyThreadState_GET(); +#else + return _PyThreadState_UncheckedGet(); +#endif +} + +// Forward declarations +void keep_alive_impl(handle nurse, handle patient); +inline PyObject *make_new_instance(PyTypeObject *type); + +class type_caster_generic { +public: + PYBIND11_NOINLINE explicit type_caster_generic(const std::type_info &type_info) + : typeinfo(get_type_info(type_info)), cpptype(&type_info) {} + + explicit type_caster_generic(const type_info *typeinfo) + : typeinfo(typeinfo), cpptype(typeinfo ? typeinfo->cpptype : nullptr) {} + + bool load(handle src, bool convert) { return load_impl<type_caster_generic>(src, convert); } + + PYBIND11_NOINLINE static handle cast(const void *_src, + return_value_policy policy, + handle parent, + const detail::type_info *tinfo, + void *(*copy_constructor)(const void *), + void *(*move_constructor)(const void *), + const void *existing_holder = nullptr) { + if (!tinfo) { // no type info: error will be set already + return handle(); + } + + void *src = const_cast<void *>(_src); + if (src == nullptr) { + return none().release(); + } + + if (handle registered_inst = find_registered_python_instance(src, tinfo)) { + return registered_inst; + } + + auto inst = reinterpret_steal<object>(make_new_instance(tinfo->type)); + auto *wrapper = reinterpret_cast<instance *>(inst.ptr()); + wrapper->owned = false; + void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); + + switch (policy) { + case return_value_policy::automatic: + case return_value_policy::take_ownership: + valueptr = src; + wrapper->owned = true; + break; + + case return_value_policy::automatic_reference: + case return_value_policy::reference: + valueptr = src; + wrapper->owned = false; + break; + + case return_value_policy::copy: + if (copy_constructor) { + valueptr = copy_constructor(src); + } else { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + std::string type_name(tinfo->cpptype->name()); + detail::clean_type_id(type_name); + throw cast_error("return_value_policy = copy, but type " + type_name + + " is non-copyable!"); +#else + throw cast_error("return_value_policy = copy, but type is " + "non-copyable! (#define PYBIND11_DETAILED_ERROR_MESSAGES or " + "compile in debug mode for details)"); +#endif + } + wrapper->owned = true; + break; + + case return_value_policy::move: + if (move_constructor) { + valueptr = move_constructor(src); + } else if (copy_constructor) { + valueptr = copy_constructor(src); + } else { +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + std::string type_name(tinfo->cpptype->name()); + detail::clean_type_id(type_name); + throw cast_error("return_value_policy = move, but type " + type_name + + " is neither movable nor copyable!"); +#else + throw cast_error("return_value_policy = move, but type is neither " + "movable nor copyable! " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in " + "debug mode for details)"); +#endif + } + wrapper->owned = true; + break; + + case return_value_policy::reference_internal: + valueptr = src; + wrapper->owned = false; + keep_alive_impl(inst, parent); + break; + + default: + throw cast_error("unhandled return_value_policy: should not happen!"); + } + + tinfo->init_instance(wrapper, existing_holder); + + return inst.release(); + } + + // Base methods for generic caster; there are overridden in copyable_holder_caster + void load_value(value_and_holder &&v_h) { + auto *&vptr = v_h.value_ptr(); + // Lazy allocation for unallocated values: + if (vptr == nullptr) { + const auto *type = v_h.type ? v_h.type : typeinfo; + if (type->operator_new) { + vptr = type->operator_new(type->type_size); + } else { +#if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) + if (type->type_align > __STDCPP_DEFAULT_NEW_ALIGNMENT__) { + vptr = ::operator new(type->type_size, std::align_val_t(type->type_align)); + } else { + vptr = ::operator new(type->type_size); + } +#else + vptr = ::operator new(type->type_size); +#endif + } + } + value = vptr; + } + bool try_implicit_casts(handle src, bool convert) { + for (const auto &cast : typeinfo->implicit_casts) { + type_caster_generic sub_caster(*cast.first); + if (sub_caster.load(src, convert)) { + value = cast.second(sub_caster.value); + return true; + } + } + return false; + } + bool try_direct_conversions(handle src) { + for (auto &converter : *typeinfo->direct_conversions) { + if (converter(src.ptr(), value)) { + return true; + } + } + return false; + } + void check_holder_compat() {} + + PYBIND11_NOINLINE static void *local_load(PyObject *src, const type_info *ti) { + auto caster = type_caster_generic(ti); + if (caster.load(src, false)) { + return caster.value; + } + return nullptr; + } + + /// Try to load with foreign typeinfo, if available. Used when there is no + /// native typeinfo, or when the native one wasn't able to produce a value. + PYBIND11_NOINLINE bool try_load_foreign_module_local(handle src) { + constexpr auto *local_key = PYBIND11_MODULE_LOCAL_ID; + const auto pytype = type::handle_of(src); + if (!hasattr(pytype, local_key)) { + return false; + } + + type_info *foreign_typeinfo = reinterpret_borrow<capsule>(getattr(pytype, local_key)); + // Only consider this foreign loader if actually foreign and is a loader of the correct cpp + // type + if (foreign_typeinfo->module_local_load == &local_load + || (cpptype && !same_type(*cpptype, *foreign_typeinfo->cpptype))) { + return false; + } + + if (auto *result = foreign_typeinfo->module_local_load(src.ptr(), foreign_typeinfo)) { + value = result; + return true; + } + return false; + } + + // Implementation of `load`; this takes the type of `this` so that it can dispatch the relevant + // bits of code between here and copyable_holder_caster where the two classes need different + // logic (without having to resort to virtual inheritance). + template <typename ThisT> + PYBIND11_NOINLINE bool load_impl(handle src, bool convert) { + if (!src) { + return false; + } + if (!typeinfo) { + return try_load_foreign_module_local(src); + } + + auto &this_ = static_cast<ThisT &>(*this); + this_.check_holder_compat(); + + PyTypeObject *srctype = Py_TYPE(src.ptr()); + + // Case 1: If src is an exact type match for the target type then we can reinterpret_cast + // the instance's value pointer to the target type: + if (srctype == typeinfo->type) { + this_.load_value(reinterpret_cast<instance *>(src.ptr())->get_value_and_holder()); + return true; + } + // Case 2: We have a derived class + if (PyType_IsSubtype(srctype, typeinfo->type)) { + const auto &bases = all_type_info(srctype); + bool no_cpp_mi = typeinfo->simple_type; + + // Case 2a: the python type is a Python-inherited derived class that inherits from just + // one simple (no MI) pybind11 class, or is an exact match, so the C++ instance is of + // the right type and we can use reinterpret_cast. + // (This is essentially the same as case 2b, but because not using multiple inheritance + // is extremely common, we handle it specially to avoid the loop iterator and type + // pointer lookup overhead) + if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { + this_.load_value(reinterpret_cast<instance *>(src.ptr())->get_value_and_holder()); + return true; + } + // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see + // if we can find an exact match (or, for a simple C++ type, an inherited match); if + // so, we can safely reinterpret_cast to the relevant pointer. + if (bases.size() > 1) { + for (auto *base : bases) { + if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) + : base->type == typeinfo->type) { + this_.load_value( + reinterpret_cast<instance *>(src.ptr())->get_value_and_holder(base)); + return true; + } + } + } + + // Case 2c: C++ multiple inheritance is involved and we couldn't find an exact type + // match in the registered bases, above, so try implicit casting (needed for proper C++ + // casting when MI is involved). + if (this_.try_implicit_casts(src, convert)) { + return true; + } + } + + // Perform an implicit conversion + if (convert) { + for (const auto &converter : typeinfo->implicit_conversions) { + auto temp = reinterpret_steal<object>(converter(src.ptr(), typeinfo->type)); + if (load_impl<ThisT>(temp, false)) { + loader_life_support::add_patient(temp); + return true; + } + } + if (this_.try_direct_conversions(src)) { + return true; + } + } + + // Failed to match local typeinfo. Try again with global. + if (typeinfo->module_local) { + if (auto *gtype = get_global_type_info(*typeinfo->cpptype)) { + typeinfo = gtype; + return load(src, false); + } + } + + // Global typeinfo has precedence over foreign module_local + if (try_load_foreign_module_local(src)) { + return true; + } + + // Custom converters didn't take None, now we convert None to nullptr. + if (src.is_none()) { + // Defer accepting None to other overloads (if we aren't in convert mode): + if (!convert) { + return false; + } + value = nullptr; + return true; + } + + return false; + } + + // Called to do type lookup and wrap the pointer and type in a pair when a dynamic_cast + // isn't needed or can't be used. If the type is unknown, sets the error and returns a pair + // with .second = nullptr. (p.first = nullptr is not an error: it becomes None). + PYBIND11_NOINLINE static std::pair<const void *, const type_info *> + src_and_type(const void *src, + const std::type_info &cast_type, + const std::type_info *rtti_type = nullptr) { + if (auto *tpi = get_type_info(cast_type)) { + return {src, const_cast<const type_info *>(tpi)}; + } + + // Not found, set error: + std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); + detail::clean_type_id(tname); + std::string msg = "Unregistered type : " + tname; + PyErr_SetString(PyExc_TypeError, msg.c_str()); + return {nullptr, nullptr}; + } + + const type_info *typeinfo = nullptr; + const std::type_info *cpptype = nullptr; + void *value = nullptr; +}; + +/** + * Determine suitable casting operator for pointer-or-lvalue-casting type casters. The type caster + * needs to provide `operator T*()` and `operator T&()` operators. + * + * If the type supports moving the value away via an `operator T&&() &&` method, it should use + * `movable_cast_op_type` instead. + */ +template <typename T> +using cast_op_type = conditional_t<std::is_pointer<remove_reference_t<T>>::value, + typename std::add_pointer<intrinsic_t<T>>::type, + typename std::add_lvalue_reference<intrinsic_t<T>>::type>; + +/** + * Determine suitable casting operator for a type caster with a movable value. Such a type caster + * needs to provide `operator T*()`, `operator T&()`, and `operator T&&() &&`. The latter will be + * called in appropriate contexts where the value can be moved rather than copied. + * + * These operator are automatically provided when using the PYBIND11_TYPE_CASTER macro. + */ +template <typename T> +using movable_cast_op_type + = conditional_t<std::is_pointer<typename std::remove_reference<T>::type>::value, + typename std::add_pointer<intrinsic_t<T>>::type, + conditional_t<std::is_rvalue_reference<T>::value, + typename std::add_rvalue_reference<intrinsic_t<T>>::type, + typename std::add_lvalue_reference<intrinsic_t<T>>::type>>; + +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template <typename Container, typename SFINAE = void> +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; + +template <typename Container> +struct container_mapped_type_traits< + Container, + typename std::enable_if< + std::is_same<typename Container::mapped_type, Container>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template <typename Container> +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation<std::is_same<typename Container::mapped_type, Container>>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template <typename Container, typename SFINAE = void> +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template <typename Container> +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same<typename Container::value_type, Container>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template <typename Container> +struct container_value_type_traits< + Container, + typename std::enable_if< + negation<std::is_same<typename Container::value_type, Container>>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template <typename T, bool is_this_a_map> +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template <typename A, typename B> +struct impl_type_to_check_recursively<std::pair<A, B>, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const<A>::type; + using if_not_recursive = std::pair<typename std::remove_const<A>::type, B>; +}; + +/* + * Implementation of `recursive_container_traits` below. + */ +template <typename Container, typename SFINAE = void> +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; + +template <typename Container> +struct impl_recursive_container_traits< + Container, + typename std::enable_if<container_value_type_traits<Container>::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits<Container>::has_recursive_mapped_type + || container_value_type_traits<Container>::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair<A const, B>`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits<Container>::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits<Container>::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template <typename Container, typename SFINAE = void> +struct recursive_container_traits : impl_recursive_container_traits<Container> {}; + +template <typename T> +struct is_move_constructible + : all_of<std::is_move_constructible<T>, + is_move_constructible< + typename recursive_container_traits<T>::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible<recursive_bottom> : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template <typename T1, typename T2> +struct is_move_constructible<std::pair<T1, T2>> + : all_of<is_move_constructible<T1>, is_move_constructible<T2>> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector<T> (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template <typename T> +struct is_copy_constructible + : all_of<std::is_copy_constructible<T>, + is_copy_constructible< + typename recursive_container_traits<T>::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible<recursive_bottom> : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the copy constructor not exist when the two types aren't +// themselves copy constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template <typename T1, typename T2> +struct is_copy_constructible<std::pair<T1, T2>> + : all_of<is_copy_constructible<T1>, is_copy_constructible<T2>> {}; + +// The same problems arise with std::is_copy_assignable, so we use the same workaround. +template <typename T> +struct is_copy_assignable + : all_of< + std::is_copy_assignable<T>, + is_copy_assignable<typename recursive_container_traits<T>::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable<recursive_bottom> : std::true_type {}; + +template <typename T1, typename T2> +struct is_copy_assignable<std::pair<T1, T2>> + : all_of<is_copy_assignable<T1>, is_copy_assignable<T2>> {}; + +PYBIND11_NAMESPACE_END(detail) + +// polymorphic_type_hook<itype>::get(src, tinfo) determines whether the object pointed +// to by `src` actually is an instance of some class derived from `itype`. +// If so, it sets `tinfo` to point to the std::type_info representing that derived +// type, and returns a pointer to the start of the most-derived object of that type +// (in which `src` is a subobject; this will be the same address as `src` in most +// single inheritance cases). If not, or if `src` is nullptr, it simply returns `src` +// and leaves `tinfo` at its default value of nullptr. +// +// The default polymorphic_type_hook just returns src. A specialization for polymorphic +// types determines the runtime type of the passed object and adjusts the this-pointer +// appropriately via dynamic_cast<void*>. This is what enables a C++ Animal* to appear +// to Python as a Dog (if Dog inherits from Animal, Animal is polymorphic, Dog is +// registered with pybind11, and this Animal is in fact a Dog). +// +// You may specialize polymorphic_type_hook yourself for types that want to appear +// polymorphic to Python but do not use C++ RTTI. (This is a not uncommon pattern +// in performance-sensitive applications, used most notably in LLVM.) +// +// polymorphic_type_hook_base allows users to specialize polymorphic_type_hook with +// std::enable_if. User provided specializations will always have higher priority than +// the default implementation and specialization provided in polymorphic_type_hook_base. +template <typename itype, typename SFINAE = void> +struct polymorphic_type_hook_base { + static const void *get(const itype *src, const std::type_info *&) { return src; } +}; +template <typename itype> +struct polymorphic_type_hook_base<itype, detail::enable_if_t<std::is_polymorphic<itype>::value>> { + static const void *get(const itype *src, const std::type_info *&type) { + type = src ? &typeid(*src) : nullptr; + return dynamic_cast<const void *>(src); + } +}; +template <typename itype, typename SFINAE = void> +struct polymorphic_type_hook : public polymorphic_type_hook_base<itype> {}; + +PYBIND11_NAMESPACE_BEGIN(detail) + +/// Generic type caster for objects stored on the heap +template <typename type> +class type_caster_base : public type_caster_generic { + using itype = intrinsic_t<type>; + +public: + static constexpr auto name = const_name<type>(); + + type_caster_base() : type_caster_base(typeid(type)) {} + explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) {} + + static handle cast(const itype &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(itype &&src, return_value_policy, handle parent) { + return cast(&src, return_value_policy::move, parent); + } + + // Returns a (pointer, type_info) pair taking care of necessary type lookup for a + // polymorphic type (using RTTI by default, but can be overridden by specializing + // polymorphic_type_hook). If the instance isn't derived, returns the base version. + static std::pair<const void *, const type_info *> src_and_type(const itype *src) { + const auto &cast_type = typeid(itype); + const std::type_info *instance_type = nullptr; + const void *vsrc = polymorphic_type_hook<itype>::get(src, instance_type); + if (instance_type && !same_type(cast_type, *instance_type)) { + // This is a base pointer to a derived type. If the derived type is registered + // with pybind11, we want to make the full derived object available. + // In the typical case where itype is polymorphic, we get the correct + // derived pointer (which may be != base pointer) by a dynamic_cast to + // most derived type. If itype is not polymorphic, we won't get here + // except via a user-provided specialization of polymorphic_type_hook, + // and the user has promised that no this-pointer adjustment is + // required in that case, so it's OK to use static_cast. + if (const auto *tpi = get_type_info(*instance_type)) { + return {vsrc, tpi}; + } + } + // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, + // so don't do a cast + return type_caster_generic::src_and_type(src, cast_type, instance_type); + } + + static handle cast(const itype *src, return_value_policy policy, handle parent) { + auto st = src_and_type(src); + return type_caster_generic::cast(st.first, + policy, + parent, + st.second, + make_copy_constructor(src), + make_move_constructor(src)); + } + + static handle cast_holder(const itype *src, const void *holder) { + auto st = src_and_type(src); + return type_caster_generic::cast(st.first, + return_value_policy::take_ownership, + {}, + st.second, + nullptr, + nullptr, + holder); + } + + template <typename T> + using cast_op_type = detail::cast_op_type<T>; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator itype *() { return (type *) value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator itype &() { + if (!value) { + throw reference_cast_error(); + } + return *((itype *) value); + } + +protected: + using Constructor = void *(*) (const void *); + + /* Only enabled when the types are {copy,move}-constructible *and* when the type + does not have a private operator new implementation. A comma operator is used in the + decltype argument to apply SFINAE to the public copy/move constructors.*/ + template <typename T, typename = enable_if_t<is_copy_constructible<T>::value>> + static auto make_copy_constructor(const T *) + -> decltype(new T(std::declval<const T>()), Constructor{}) { + return [](const void *arg) -> void * { return new T(*reinterpret_cast<const T *>(arg)); }; + } + + template <typename T, typename = enable_if_t<is_move_constructible<T>::value>> + static auto make_move_constructor(const T *) + -> decltype(new T(std::declval<T &&>()), Constructor{}) { + return [](const void *arg) -> void * { + return new T(std::move(*const_cast<T *>(reinterpret_cast<const T *>(arg)))); + }; + } + + static Constructor make_copy_constructor(...) { return nullptr; } + static Constructor make_move_constructor(...) { return nullptr; } +}; + +PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { + if (auto *type_data = get_type_info(ti)) { + handle th((PyObject *) type_data->type); + return th.attr("__module__").cast<std::string>() + '.' + + th.attr("__qualname__").cast<std::string>(); + } + return clean_type_id(ti.name()); +} + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/typeid.h b/include/pybind11/detail/typeid.h index 148889ff..a67b5213 100644 --- a/include/pybind11/detail/typeid.h +++ b/include/pybind11/detail/typeid.h @@ -13,29 +13,33 @@ #include <cstdlib> #if defined(__GNUG__) -#include <cxxabi.h> +# include <cxxabi.h> #endif #include "common.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) + /// Erase all occurrences of a substring inline void erase_all(std::string &string, const std::string &search) { for (size_t pos = 0;;) { pos = string.find(search, pos); - if (pos == std::string::npos) break; + if (pos == std::string::npos) { + break; + } string.erase(pos, search.length()); } } -PYBIND11_NOINLINE inline void clean_type_id(std::string &name) { +PYBIND11_NOINLINE void clean_type_id(std::string &name) { #if defined(__GNUG__) int status = 0; - std::unique_ptr<char, void (*)(void *)> res { - abi::__cxa_demangle(name.c_str(), nullptr, nullptr, &status), std::free }; - if (status == 0) + std::unique_ptr<char, void (*)(void *)> res{ + abi::__cxa_demangle(name.c_str(), nullptr, nullptr, &status), std::free}; + if (status == 0) { name = res.get(); + } #else detail::erase_all(name, "class "); detail::erase_all(name, "struct "); @@ -43,13 +47,19 @@ PYBIND11_NOINLINE inline void clean_type_id(std::string &name) { #endif detail::erase_all(name, "pybind11::"); } -PYBIND11_NAMESPACE_END(detail) -/// Return a string representation of a C++ type -template <typename T> static std::string type_id() { - std::string name(typeid(T).name()); +inline std::string clean_type_id(const char *typeid_name) { + std::string name(typeid_name); detail::clean_type_id(name); return name; } +PYBIND11_NAMESPACE_END(detail) + +/// Return a string representation of a C++ type +template <typename T> +static std::string type_id() { + return detail::clean_type_id(typeid(T).name()); +} + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/eigen.h b/include/pybind11/eigen.h index e8c6f633..273b9c93 100644 --- a/include/pybind11/eigen.h +++ b/include/pybind11/eigen.h @@ -9,599 +9,4 @@ #pragma once -#include "numpy.h" - -#if defined(__INTEL_COMPILER) -# pragma warning(disable: 1682) // implicit conversion of a 64-bit integral type to a smaller integral type (potential portability problem) -#elif defined(__GNUG__) || defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wconversion" -# pragma GCC diagnostic ignored "-Wdeprecated-declarations" -# ifdef __clang__ -// Eigen generates a bunch of implicit-copy-constructor-is-deprecated warnings with -Wdeprecated -// under Clang, so disable that warning here: -# pragma GCC diagnostic ignored "-Wdeprecated" -# endif -# if __GNUC__ >= 7 -# pragma GCC diagnostic ignored "-Wint-in-bool-context" -# endif -#endif - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant -# pragma warning(disable: 4996) // warning C4996: std::unary_negate is deprecated in C++17 -#endif - -#include <Eigen/Core> -#include <Eigen/SparseCore> - -// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit -// move constructors that break things. We could detect this an explicitly copy, but an extra copy -// of matrices seems highly undesirable. -static_assert(EIGEN_VERSION_AT_LEAST(3,2,7), "Eigen support in pybind11 requires Eigen >= 3.2.7"); - -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) - -// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: -using EigenDStride = Eigen::Stride<Eigen::Dynamic, Eigen::Dynamic>; -template <typename MatrixType> using EigenDRef = Eigen::Ref<MatrixType, 0, EigenDStride>; -template <typename MatrixType> using EigenDMap = Eigen::Map<MatrixType, 0, EigenDStride>; - -PYBIND11_NAMESPACE_BEGIN(detail) - -#if EIGEN_VERSION_AT_LEAST(3,3,0) -using EigenIndex = Eigen::Index; -#else -using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; -#endif - -// Matches Eigen::Map, Eigen::Ref, blocks, etc: -template <typename T> using is_eigen_dense_map = all_of<is_template_base_of<Eigen::DenseBase, T>, std::is_base_of<Eigen::MapBase<T, Eigen::ReadOnlyAccessors>, T>>; -template <typename T> using is_eigen_mutable_map = std::is_base_of<Eigen::MapBase<T, Eigen::WriteAccessors>, T>; -template <typename T> using is_eigen_dense_plain = all_of<negation<is_eigen_dense_map<T>>, is_template_base_of<Eigen::PlainObjectBase, T>>; -template <typename T> using is_eigen_sparse = is_template_base_of<Eigen::SparseMatrixBase, T>; -// Test for objects inheriting from EigenBase<Derived> that aren't captured by the above. This -// basically covers anything that can be assigned to a dense matrix but that don't have a typical -// matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and -// SelfAdjointView fall into this category. -template <typename T> using is_eigen_other = all_of< - is_template_base_of<Eigen::EigenBase, T>, - negation<any_of<is_eigen_dense_map<T>, is_eigen_dense_plain<T>, is_eigen_sparse<T>>> ->; - -// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): -template <bool EigenRowMajor> struct EigenConformable { - bool conformable = false; - EigenIndex rows = 0, cols = 0; - EigenDStride stride{0, 0}; // Only valid if negativestrides is false! - bool negativestrides = false; // If true, do not use stride! - - EigenConformable(bool fits = false) : conformable{fits} {} - // Matrix type: - EigenConformable(EigenIndex r, EigenIndex c, - EigenIndex rstride, EigenIndex cstride) : - conformable{true}, rows{r}, cols{c} { - // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747 - if (rstride < 0 || cstride < 0) { - negativestrides = true; - } else { - stride = {EigenRowMajor ? rstride : cstride /* outer stride */, - EigenRowMajor ? cstride : rstride /* inner stride */ }; - } - } - // Vector type: - EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) - : EigenConformable(r, c, r == 1 ? c*stride : stride, c == 1 ? r : r*stride) {} - - template <typename props> bool stride_compatible() const { - // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, - // matching strides, or a dimension size of 1 (in which case the stride value is irrelevant) - return - !negativestrides && - (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() || - (EigenRowMajor ? cols : rows) == 1) && - (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() || - (EigenRowMajor ? rows : cols) == 1); - } - operator bool() const { return conformable; } -}; - -template <typename Type> struct eigen_extract_stride { using type = Type; }; -template <typename PlainObjectType, int MapOptions, typename StrideType> -struct eigen_extract_stride<Eigen::Map<PlainObjectType, MapOptions, StrideType>> { using type = StrideType; }; -template <typename PlainObjectType, int Options, typename StrideType> -struct eigen_extract_stride<Eigen::Ref<PlainObjectType, Options, StrideType>> { using type = StrideType; }; - -// Helper struct for extracting information from an Eigen type -template <typename Type_> struct EigenProps { - using Type = Type_; - using Scalar = typename Type::Scalar; - using StrideType = typename eigen_extract_stride<Type>::type; - static constexpr EigenIndex - rows = Type::RowsAtCompileTime, - cols = Type::ColsAtCompileTime, - size = Type::SizeAtCompileTime; - static constexpr bool - row_major = Type::IsRowMajor, - vector = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 - fixed_rows = rows != Eigen::Dynamic, - fixed_cols = cols != Eigen::Dynamic, - fixed = size != Eigen::Dynamic, // Fully-fixed size - dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size - - template <EigenIndex i, EigenIndex ifzero> using if_zero = std::integral_constant<EigenIndex, i == 0 ? ifzero : i>; - static constexpr EigenIndex inner_stride = if_zero<StrideType::InnerStrideAtCompileTime, 1>::value, - outer_stride = if_zero<StrideType::OuterStrideAtCompileTime, - vector ? size : row_major ? cols : rows>::value; - static constexpr bool dynamic_stride = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; - static constexpr bool requires_row_major = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; - static constexpr bool requires_col_major = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; - - // Takes an input array and determines whether we can make it fit into the Eigen type. If - // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector - // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). - static EigenConformable<row_major> conformable(const array &a) { - const auto dims = a.ndim(); - if (dims < 1 || dims > 2) - return false; - - if (dims == 2) { // Matrix type: require exact match (or dynamic) - - EigenIndex - np_rows = a.shape(0), - np_cols = a.shape(1), - np_rstride = a.strides(0) / static_cast<ssize_t>(sizeof(Scalar)), - np_cstride = a.strides(1) / static_cast<ssize_t>(sizeof(Scalar)); - if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) - return false; - - return {np_rows, np_cols, np_rstride, np_cstride}; - } - - // Otherwise we're storing an n-vector. Only one of the strides will be used, but whichever - // is used, we want the (single) numpy stride value. - const EigenIndex n = a.shape(0), - stride = a.strides(0) / static_cast<ssize_t>(sizeof(Scalar)); - - if (vector) { // Eigen type is a compile-time vector - if (fixed && size != n) - return false; // Vector size mismatch - return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; - } - else if (fixed) { - // The type has a fixed size, but is not a vector: abort - return false; - } - else if (fixed_cols) { - // Since this isn't a vector, cols must be != 1. We allow this only if it exactly - // equals the number of elements (rows is Dynamic, and so 1 row is allowed). - if (cols != n) return false; - return {1, n, stride}; - } - else { - // Otherwise it's either fully dynamic, or column dynamic; both become a column vector - if (fixed_rows && rows != n) return false; - return {n, 1, stride}; - } - } - - static constexpr bool show_writeable = is_eigen_dense_map<Type>::value && is_eigen_mutable_map<Type>::value; - static constexpr bool show_order = is_eigen_dense_map<Type>::value; - static constexpr bool show_c_contiguous = show_order && requires_row_major; - static constexpr bool show_f_contiguous = !show_c_contiguous && show_order && requires_col_major; - - static constexpr auto descriptor = - _("numpy.ndarray[") + npy_format_descriptor<Scalar>::name + - _("[") + _<fixed_rows>(_<(size_t) rows>(), _("m")) + - _(", ") + _<fixed_cols>(_<(size_t) cols>(), _("n")) + - _("]") + - // For a reference type (e.g. Ref<MatrixXd>) we have other constraints that might need to be - // satisfied: writeable=True (for a mutable reference), and, depending on the map's stride - // options, possibly f_contiguous or c_contiguous. We include them in the descriptor output - // to provide some hint as to why a TypeError is occurring (otherwise it can be confusing to - // see that a function accepts a 'numpy.ndarray[float64[3,2]]' and an error message that you - // *gave* a numpy.ndarray of the right type and dimensions. - _<show_writeable>(", flags.writeable", "") + - _<show_c_contiguous>(", flags.c_contiguous", "") + - _<show_f_contiguous>(", flags.f_contiguous", "") + - _("]"); -}; - -// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, -// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. -template <typename props> handle eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { - constexpr ssize_t elem_size = sizeof(typename props::Scalar); - array a; - if (props::vector) - a = array({ src.size() }, { elem_size * src.innerStride() }, src.data(), base); - else - a = array({ src.rows(), src.cols() }, { elem_size * src.rowStride(), elem_size * src.colStride() }, - src.data(), base); - - if (!writeable) - array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; - - return a.release(); -} - -// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that -// reference the Eigen object's data with `base` as the python-registered base class (if omitted, -// the base will be set to None, and lifetime management is up to the caller). The numpy array is -// non-writeable if the given type is const. -template <typename props, typename Type> -handle eigen_ref_array(Type &src, handle parent = none()) { - // none here is to get past array's should-we-copy detection, which currently always - // copies when there is no base. Setting the base to None should be harmless. - return eigen_array_cast<props>(src, parent, !std::is_const<Type>::value); -} - -// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a numpy -// array that references the encapsulated data with a python-side reference to the capsule to tie -// its destruction to that of any dependent python objects. Const-ness is determined by whether or -// not the Type of the pointer given is const. -template <typename props, typename Type, typename = enable_if_t<is_eigen_dense_plain<Type>::value>> -handle eigen_encapsulate(Type *src) { - capsule base(src, [](void *o) { delete static_cast<Type *>(o); }); - return eigen_ref_array<props>(*src, base); -} - -// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense -// types. -template<typename Type> -struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> { - using Scalar = typename Type::Scalar; - using props = EigenProps<Type>; - - bool load(handle src, bool convert) { - // If we're in no-convert mode, only load if given an array of the correct type - if (!convert && !isinstance<array_t<Scalar>>(src)) - return false; - - // Coerce into an array, but don't do type conversion yet; the copy below handles it. - auto buf = array::ensure(src); - - if (!buf) - return false; - - auto dims = buf.ndim(); - if (dims < 1 || dims > 2) - return false; - - auto fits = props::conformable(buf); - if (!fits) - return false; - - // Allocate the new type, then build a numpy reference into it - value = Type(fits.rows, fits.cols); - auto ref = reinterpret_steal<array>(eigen_ref_array<props>(value)); - if (dims == 1) ref = ref.squeeze(); - else if (ref.ndim() == 1) buf = buf.squeeze(); - - int result = detail::npy_api::get().PyArray_CopyInto_(ref.ptr(), buf.ptr()); - - if (result < 0) { // Copy failed! - PyErr_Clear(); - return false; - } - - return true; - } - -private: - - // Cast implementation - template <typename CType> - static handle cast_impl(CType *src, return_value_policy policy, handle parent) { - switch (policy) { - case return_value_policy::take_ownership: - case return_value_policy::automatic: - return eigen_encapsulate<props>(src); - case return_value_policy::move: - return eigen_encapsulate<props>(new CType(std::move(*src))); - case return_value_policy::copy: - return eigen_array_cast<props>(*src); - case return_value_policy::reference: - case return_value_policy::automatic_reference: - return eigen_ref_array<props>(*src); - case return_value_policy::reference_internal: - return eigen_ref_array<props>(*src, parent); - default: - throw cast_error("unhandled return_value_policy: should not happen!"); - }; - } - -public: - - // Normal returned non-reference, non-const value: - static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { - return cast_impl(&src, return_value_policy::move, parent); - } - // If you return a non-reference const, we mark the numpy array readonly: - static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { - return cast_impl(&src, return_value_policy::move, parent); - } - // lvalue reference return; default (automatic) becomes copy - static handle cast(Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) - policy = return_value_policy::copy; - return cast_impl(&src, policy, parent); - } - // const lvalue reference return; default (automatic) becomes copy - static handle cast(const Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic || policy == return_value_policy::automatic_reference) - policy = return_value_policy::copy; - return cast(&src, policy, parent); - } - // non-const pointer return - static handle cast(Type *src, return_value_policy policy, handle parent) { - return cast_impl(src, policy, parent); - } - // const pointer return - static handle cast(const Type *src, return_value_policy policy, handle parent) { - return cast_impl(src, policy, parent); - } - - static constexpr auto name = props::descriptor; - - operator Type*() { return &value; } - operator Type&() { return value; } - operator Type&&() && { return std::move(value); } - template <typename T> using cast_op_type = movable_cast_op_type<T>; - -private: - Type value; -}; - -// Base class for casting reference/map/block/etc. objects back to python. -template <typename MapType> struct eigen_map_caster { -private: - using props = EigenProps<MapType>; - -public: - - // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has - // to stay around), but we'll allow it under the assumption that you know what you're doing (and - // have an appropriate keep_alive in place). We return a numpy array pointing directly at the - // ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) Note - // that this means you need to ensure you don't destroy the object in some other way (e.g. with - // an appropriate keep_alive, or with a reference to a statically allocated matrix). - static handle cast(const MapType &src, return_value_policy policy, handle parent) { - switch (policy) { - case return_value_policy::copy: - return eigen_array_cast<props>(src); - case return_value_policy::reference_internal: - return eigen_array_cast<props>(src, parent, is_eigen_mutable_map<MapType>::value); - case return_value_policy::reference: - case return_value_policy::automatic: - case return_value_policy::automatic_reference: - return eigen_array_cast<props>(src, none(), is_eigen_mutable_map<MapType>::value); - default: - // move, take_ownership don't make any sense for a ref/map: - pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); - } - } - - static constexpr auto name = props::descriptor; - - // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return - // types but not bound arguments). We still provide them (with an explicitly delete) so that - // you end up here if you try anyway. - bool load(handle, bool) = delete; - operator MapType() = delete; - template <typename> using cast_op_type = MapType; -}; - -// We can return any map-like object (but can only load Refs, specialized next): -template <typename Type> struct type_caster<Type, enable_if_t<is_eigen_dense_map<Type>::value>> - : eigen_map_caster<Type> {}; - -// Loader for Ref<...> arguments. See the documentation for info on how to make this work without -// copying (it requires some extra effort in many cases). -template <typename PlainObjectType, typename StrideType> -struct type_caster< - Eigen::Ref<PlainObjectType, 0, StrideType>, - enable_if_t<is_eigen_dense_map<Eigen::Ref<PlainObjectType, 0, StrideType>>::value> -> : public eigen_map_caster<Eigen::Ref<PlainObjectType, 0, StrideType>> { -private: - using Type = Eigen::Ref<PlainObjectType, 0, StrideType>; - using props = EigenProps<Type>; - using Scalar = typename props::Scalar; - using MapType = Eigen::Map<PlainObjectType, 0, StrideType>; - using Array = array_t<Scalar, array::forcecast | - ((props::row_major ? props::inner_stride : props::outer_stride) == 1 ? array::c_style : - (props::row_major ? props::outer_stride : props::inner_stride) == 1 ? array::f_style : 0)>; - static constexpr bool need_writeable = is_eigen_mutable_map<Type>::value; - // Delay construction (these have no default constructor) - std::unique_ptr<MapType> map; - std::unique_ptr<Type> ref; - // Our array. When possible, this is just a numpy array pointing to the source data, but - // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an incompatible - // layout, or is an array of a type that needs to be converted). Using a numpy temporary - // (rather than an Eigen temporary) saves an extra copy when we need both type conversion and - // storage order conversion. (Note that we refuse to use this temporary copy when loading an - // argument for a Ref<M> with M non-const, i.e. a read-write reference). - Array copy_or_ref; -public: - bool load(handle src, bool convert) { - // First check whether what we have is already an array of the right type. If not, we can't - // avoid a copy (because the copy is also going to do type conversion). - bool need_copy = !isinstance<Array>(src); - - EigenConformable<props::row_major> fits; - if (!need_copy) { - // We don't need a converting copy, but we also need to check whether the strides are - // compatible with the Ref's stride requirements - auto aref = reinterpret_borrow<Array>(src); - - if (aref && (!need_writeable || aref.writeable())) { - fits = props::conformable(aref); - if (!fits) return false; // Incompatible dimensions - if (!fits.template stride_compatible<props>()) - need_copy = true; - else - copy_or_ref = std::move(aref); - } - else { - need_copy = true; - } - } - - if (need_copy) { - // We need to copy: If we need a mutable reference, or we're not supposed to convert - // (either because we're in the no-convert overload pass, or because we're explicitly - // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. - if (!convert || need_writeable) return false; - - Array copy = Array::ensure(src); - if (!copy) return false; - fits = props::conformable(copy); - if (!fits || !fits.template stride_compatible<props>()) - return false; - copy_or_ref = std::move(copy); - loader_life_support::add_patient(copy_or_ref); - } - - ref.reset(); - map.reset(new MapType(data(copy_or_ref), fits.rows, fits.cols, make_stride(fits.stride.outer(), fits.stride.inner()))); - ref.reset(new Type(*map)); - - return true; - } - - operator Type*() { return ref.get(); } - operator Type&() { return *ref; } - template <typename _T> using cast_op_type = pybind11::detail::cast_op_type<_T>; - -private: - template <typename T = Type, enable_if_t<is_eigen_mutable_map<T>::value, int> = 0> - Scalar *data(Array &a) { return a.mutable_data(); } - - template <typename T = Type, enable_if_t<!is_eigen_mutable_map<T>::value, int> = 0> - const Scalar *data(Array &a) { return a.data(); } - - // Attempt to figure out a constructor of `Stride` that will work. - // If both strides are fixed, use a default constructor: - template <typename S> using stride_ctor_default = bool_constant< - S::InnerStrideAtCompileTime != Eigen::Dynamic && S::OuterStrideAtCompileTime != Eigen::Dynamic && - std::is_default_constructible<S>::value>; - // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like - // Eigen::Stride, and use it: - template <typename S> using stride_ctor_dual = bool_constant< - !stride_ctor_default<S>::value && std::is_constructible<S, EigenIndex, EigenIndex>::value>; - // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use - // it (passing whichever stride is dynamic). - template <typename S> using stride_ctor_outer = bool_constant< - !any_of<stride_ctor_default<S>, stride_ctor_dual<S>>::value && - S::OuterStrideAtCompileTime == Eigen::Dynamic && S::InnerStrideAtCompileTime != Eigen::Dynamic && - std::is_constructible<S, EigenIndex>::value>; - template <typename S> using stride_ctor_inner = bool_constant< - !any_of<stride_ctor_default<S>, stride_ctor_dual<S>>::value && - S::InnerStrideAtCompileTime == Eigen::Dynamic && S::OuterStrideAtCompileTime != Eigen::Dynamic && - std::is_constructible<S, EigenIndex>::value>; - - template <typename S = StrideType, enable_if_t<stride_ctor_default<S>::value, int> = 0> - static S make_stride(EigenIndex, EigenIndex) { return S(); } - template <typename S = StrideType, enable_if_t<stride_ctor_dual<S>::value, int> = 0> - static S make_stride(EigenIndex outer, EigenIndex inner) { return S(outer, inner); } - template <typename S = StrideType, enable_if_t<stride_ctor_outer<S>::value, int> = 0> - static S make_stride(EigenIndex outer, EigenIndex) { return S(outer); } - template <typename S = StrideType, enable_if_t<stride_ctor_inner<S>::value, int> = 0> - static S make_stride(EigenIndex, EigenIndex inner) { return S(inner); } - -}; - -// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not -// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). -// load() is not supported, but we can cast them into the python domain by first copying to a -// regular Eigen::Matrix, then casting that. -template <typename Type> -struct type_caster<Type, enable_if_t<is_eigen_other<Type>::value>> { -protected: - using Matrix = Eigen::Matrix<typename Type::Scalar, Type::RowsAtCompileTime, Type::ColsAtCompileTime>; - using props = EigenProps<Matrix>; -public: - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - handle h = eigen_encapsulate<props>(new Matrix(src)); - return h; - } - static handle cast(const Type *src, return_value_policy policy, handle parent) { return cast(*src, policy, parent); } - - static constexpr auto name = props::descriptor; - - // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return - // types but not bound arguments). We still provide them (with an explicitly delete) so that - // you end up here if you try anyway. - bool load(handle, bool) = delete; - operator Type() = delete; - template <typename> using cast_op_type = Type; -}; - -template<typename Type> -struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> { - using Scalar = typename Type::Scalar; - using StorageIndex = remove_reference_t<decltype(*std::declval<Type>().outerIndexPtr())>; - using Index = typename Type::Index; - static constexpr bool rowMajor = Type::IsRowMajor; - - bool load(handle src, bool) { - if (!src) - return false; - - auto obj = reinterpret_borrow<object>(src); - object sparse_module = module_::import("scipy.sparse"); - object matrix_type = sparse_module.attr( - rowMajor ? "csr_matrix" : "csc_matrix"); - - if (!type::handle_of(obj).is(matrix_type)) { - try { - obj = matrix_type(obj); - } catch (const error_already_set &) { - return false; - } - } - - auto values = array_t<Scalar>((object) obj.attr("data")); - auto innerIndices = array_t<StorageIndex>((object) obj.attr("indices")); - auto outerIndices = array_t<StorageIndex>((object) obj.attr("indptr")); - auto shape = pybind11::tuple((pybind11::object) obj.attr("shape")); - auto nnz = obj.attr("nnz").cast<Index>(); - - if (!values || !innerIndices || !outerIndices) - return false; - - value = Eigen::MappedSparseMatrix<Scalar, Type::Flags, StorageIndex>( - shape[0].cast<Index>(), shape[1].cast<Index>(), nnz, - outerIndices.mutable_data(), innerIndices.mutable_data(), values.mutable_data()); - - return true; - } - - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - const_cast<Type&>(src).makeCompressed(); - - object matrix_type = module_::import("scipy.sparse").attr( - rowMajor ? "csr_matrix" : "csc_matrix"); - - array data(src.nonZeros(), src.valuePtr()); - array outerIndices((rowMajor ? src.rows() : src.cols()) + 1, src.outerIndexPtr()); - array innerIndices(src.nonZeros(), src.innerIndexPtr()); - - return matrix_type( - std::make_tuple(data, innerIndices, outerIndices), - std::make_pair(src.rows(), src.cols()) - ).release(); - } - - PYBIND11_TYPE_CASTER(Type, _<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", "scipy.sparse.csc_matrix[") - + npy_format_descriptor<Scalar>::name + _("]")); -}; - -PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(__GNUG__) || defined(__clang__) -# pragma GCC diagnostic pop -#elif defined(_MSC_VER) -# pragma warning(pop) -#endif +#include "eigen/matrix.h" diff --git a/include/pybind11/eigen/common.h b/include/pybind11/eigen/common.h new file mode 100644 index 00000000..24f56d15 --- /dev/null +++ b/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h new file mode 100644 index 00000000..8d4342f8 --- /dev/null +++ b/include/pybind11/eigen/matrix.h @@ -0,0 +1,714 @@ +/* + pybind11/eigen/matrix.h: Transparent conversion for dense and sparse Eigen matrices + + Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch> + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../numpy.h" +#include "common.h" + +/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. + See also: + https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir + https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler +*/ +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 +// C5054: operator '&': deprecated between enumerations of different types +#if defined(__MINGW32__) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif + +#include <Eigen/Core> +#include <Eigen/SparseCore> + +PYBIND11_WARNING_POP + +// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit +// move constructors that break things. We could detect this an explicitly copy, but an extra copy +// of matrices seems highly undesirable. +static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), + "Eigen matrix support in pybind11 requires Eigen >= 3.2.7"); + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: +using EigenDStride = Eigen::Stride<Eigen::Dynamic, Eigen::Dynamic>; +template <typename MatrixType> +using EigenDRef = Eigen::Ref<MatrixType, 0, EigenDStride>; +template <typename MatrixType> +using EigenDMap = Eigen::Map<MatrixType, 0, EigenDStride>; + +PYBIND11_NAMESPACE_BEGIN(detail) + +#if EIGEN_VERSION_AT_LEAST(3, 3, 0) +using EigenIndex = Eigen::Index; +template <typename Scalar, int Flags, typename StorageIndex> +using EigenMapSparseMatrix = Eigen::Map<Eigen::SparseMatrix<Scalar, Flags, StorageIndex>>; +#else +using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; +template <typename Scalar, int Flags, typename StorageIndex> +using EigenMapSparseMatrix = Eigen::MappedSparseMatrix<Scalar, Flags, StorageIndex>; +#endif + +// Matches Eigen::Map, Eigen::Ref, blocks, etc: +template <typename T> +using is_eigen_dense_map = all_of<is_template_base_of<Eigen::DenseBase, T>, + std::is_base_of<Eigen::MapBase<T, Eigen::ReadOnlyAccessors>, T>>; +template <typename T> +using is_eigen_mutable_map = std::is_base_of<Eigen::MapBase<T, Eigen::WriteAccessors>, T>; +template <typename T> +using is_eigen_dense_plain + = all_of<negation<is_eigen_dense_map<T>>, is_template_base_of<Eigen::PlainObjectBase, T>>; +template <typename T> +using is_eigen_sparse = is_template_base_of<Eigen::SparseMatrixBase, T>; +// Test for objects inheriting from EigenBase<Derived> that aren't captured by the above. This +// basically covers anything that can be assigned to a dense matrix but that don't have a typical +// matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and +// SelfAdjointView fall into this category. +template <typename T> +using is_eigen_other + = all_of<is_template_base_of<Eigen::EigenBase, T>, + negation<any_of<is_eigen_dense_map<T>, is_eigen_dense_plain<T>, is_eigen_sparse<T>>>>; + +// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): +template <bool EigenRowMajor> +struct EigenConformable { + bool conformable = false; + EigenIndex rows = 0, cols = 0; + EigenDStride stride{0, 0}; // Only valid if negativestrides is false! + bool negativestrides = false; // If true, do not use stride! + + // NOLINTNEXTLINE(google-explicit-constructor) + EigenConformable(bool fits = false) : conformable{fits} {} + // Matrix type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex rstride, EigenIndex cstride) + : conformable{true}, rows{r}, cols{c}, + // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. + // http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747 + stride{EigenRowMajor ? (rstride > 0 ? rstride : 0) + : (cstride > 0 ? cstride : 0) /* outer stride */, + EigenRowMajor ? (cstride > 0 ? cstride : 0) + : (rstride > 0 ? rstride : 0) /* inner stride */}, + negativestrides{rstride < 0 || cstride < 0} {} + // Vector type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) + : EigenConformable(r, c, r == 1 ? c * stride : stride, c == 1 ? r : r * stride) {} + + template <typename props> + bool stride_compatible() const { + // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, + // matching strides, or a dimension size of 1 (in which case the stride value is + // irrelevant). Alternatively, if any dimension size is 0, the strides are not relevant + // (and numpy ≥ 1.23 sets the strides to 0 in that case, so we need to check explicitly). + if (negativestrides) { + return false; + } + if (rows == 0 || cols == 0) { + return true; + } + return (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() + || (EigenRowMajor ? cols : rows) == 1) + && (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() + || (EigenRowMajor ? rows : cols) == 1); + } + // NOLINTNEXTLINE(google-explicit-constructor) + operator bool() const { return conformable; } +}; + +template <typename Type> +struct eigen_extract_stride { + using type = Type; +}; +template <typename PlainObjectType, int MapOptions, typename StrideType> +struct eigen_extract_stride<Eigen::Map<PlainObjectType, MapOptions, StrideType>> { + using type = StrideType; +}; +template <typename PlainObjectType, int Options, typename StrideType> +struct eigen_extract_stride<Eigen::Ref<PlainObjectType, Options, StrideType>> { + using type = StrideType; +}; + +// Helper struct for extracting information from an Eigen type +template <typename Type_> +struct EigenProps { + using Type = Type_; + using Scalar = typename Type::Scalar; + using StrideType = typename eigen_extract_stride<Type>::type; + static constexpr EigenIndex rows = Type::RowsAtCompileTime, cols = Type::ColsAtCompileTime, + size = Type::SizeAtCompileTime; + static constexpr bool row_major = Type::IsRowMajor, + vector + = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 + fixed_rows = rows != Eigen::Dynamic, fixed_cols = cols != Eigen::Dynamic, + fixed = size != Eigen::Dynamic, // Fully-fixed size + dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size + + template <EigenIndex i, EigenIndex ifzero> + using if_zero = std::integral_constant<EigenIndex, i == 0 ? ifzero : i>; + static constexpr EigenIndex inner_stride + = if_zero<StrideType::InnerStrideAtCompileTime, 1>::value, + outer_stride = if_zero < StrideType::OuterStrideAtCompileTime, + vector ? size + : row_major ? cols + : rows > ::value; + static constexpr bool dynamic_stride + = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; + static constexpr bool requires_row_major + = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; + static constexpr bool requires_col_major + = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; + + // Takes an input array and determines whether we can make it fit into the Eigen type. If + // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector + // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). + static EigenConformable<row_major> conformable(const array &a) { + const auto dims = a.ndim(); + if (dims < 1 || dims > 2) { + return false; + } + + if (dims == 2) { // Matrix type: require exact match (or dynamic) + + EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), + np_rstride = a.strides(0) / static_cast<ssize_t>(sizeof(Scalar)), + np_cstride = a.strides(1) / static_cast<ssize_t>(sizeof(Scalar)); + if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) { + return false; + } + + return {np_rows, np_cols, np_rstride, np_cstride}; + } + + // Otherwise we're storing an n-vector. Only one of the strides will be used, but + // whichever is used, we want the (single) numpy stride value. + const EigenIndex n = a.shape(0), + stride = a.strides(0) / static_cast<ssize_t>(sizeof(Scalar)); + + if (vector) { // Eigen type is a compile-time vector + if (fixed && size != n) { + return false; // Vector size mismatch + } + return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; + } + if (fixed) { + // The type has a fixed size, but is not a vector: abort + return false; + } + if (fixed_cols) { + // Since this isn't a vector, cols must be != 1. We allow this only if it exactly + // equals the number of elements (rows is Dynamic, and so 1 row is allowed). + if (cols != n) { + return false; + } + return {1, n, stride}; + } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector + if (fixed_rows && rows != n) { + return false; + } + return {n, 1, stride}; + } + + static constexpr bool show_writeable + = is_eigen_dense_map<Type>::value && is_eigen_mutable_map<Type>::value; + static constexpr bool show_order = is_eigen_dense_map<Type>::value; + static constexpr bool show_c_contiguous = show_order && requires_row_major; + static constexpr bool show_f_contiguous + = !show_c_contiguous && show_order && requires_col_major; + + static constexpr auto descriptor + = const_name("numpy.ndarray[") + npy_format_descriptor<Scalar>::name + const_name("[") + + const_name<fixed_rows>(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") + + const_name<fixed_cols>(const_name<(size_t) cols>(), const_name("n")) + const_name("]") + + + // For a reference type (e.g. Ref<MatrixXd>) we have other constraints that might need to + // be satisfied: writeable=True (for a mutable reference), and, depending on the map's + // stride options, possibly f_contiguous or c_contiguous. We include them in the + // descriptor output to provide some hint as to why a TypeError is occurring (otherwise + // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and + // an error message that you *gave* a numpy.ndarray of the right type and dimensions. + const_name<show_writeable>(", flags.writeable", "") + + const_name<show_c_contiguous>(", flags.c_contiguous", "") + + const_name<show_f_contiguous>(", flags.f_contiguous", "") + const_name("]"); +}; + +// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, +// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. +template <typename props> +handle +eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { + constexpr ssize_t elem_size = sizeof(typename props::Scalar); + array a; + if (props::vector) { + a = array({src.size()}, {elem_size * src.innerStride()}, src.data(), base); + } else { + a = array({src.rows(), src.cols()}, + {elem_size * src.rowStride(), elem_size * src.colStride()}, + src.data(), + base); + } + + if (!writeable) { + array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return a.release(); +} + +// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that +// reference the Eigen object's data with `base` as the python-registered base class (if omitted, +// the base will be set to None, and lifetime management is up to the caller). The numpy array is +// non-writeable if the given type is const. +template <typename props, typename Type> +handle eigen_ref_array(Type &src, handle parent = none()) { + // none here is to get past array's should-we-copy detection, which currently always + // copies when there is no base. Setting the base to None should be harmless. + return eigen_array_cast<props>(src, parent, !std::is_const<Type>::value); +} + +// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a +// numpy array that references the encapsulated data with a python-side reference to the capsule to +// tie its destruction to that of any dependent python objects. Const-ness is determined by +// whether or not the Type of the pointer given is const. +template <typename props, typename Type, typename = enable_if_t<is_eigen_dense_plain<Type>::value>> +handle eigen_encapsulate(Type *src) { + capsule base(src, [](void *o) { delete static_cast<Type *>(o); }); + return eigen_ref_array<props>(*src, base); +} + +// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense +// types. +template <typename Type> +struct type_caster<Type, enable_if_t<is_eigen_dense_plain<Type>::value>> { + using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer<Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using props = EigenProps<Type>; + + bool load(handle src, bool convert) { + // If we're in no-convert mode, only load if given an array of the correct type + if (!convert && !isinstance<array_t<Scalar>>(src)) { + return false; + } + + // Coerce into an array, but don't do type conversion yet; the copy below handles it. + auto buf = array::ensure(src); + + if (!buf) { + return false; + } + + auto dims = buf.ndim(); + if (dims < 1 || dims > 2) { + return false; + } + + auto fits = props::conformable(buf); + if (!fits) { + return false; + } + + // Allocate the new type, then build a numpy reference into it + value = Type(fits.rows, fits.cols); + auto ref = reinterpret_steal<array>(eigen_ref_array<props>(value)); + if (dims == 1) { + ref = ref.squeeze(); + } else if (ref.ndim() == 1) { + buf = buf.squeeze(); + } + + int result = detail::npy_api::get().PyArray_CopyInto_(ref.ptr(), buf.ptr()); + + if (result < 0) { // Copy failed! + PyErr_Clear(); + return false; + } + + return true; + } + +private: + // Cast implementation + template <typename CType> + static handle cast_impl(CType *src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::take_ownership: + case return_value_policy::automatic: + return eigen_encapsulate<props>(src); + case return_value_policy::move: + return eigen_encapsulate<props>(new CType(std::move(*src))); + case return_value_policy::copy: + return eigen_array_cast<props>(*src); + case return_value_policy::reference: + case return_value_policy::automatic_reference: + return eigen_ref_array<props>(*src); + case return_value_policy::reference_internal: + return eigen_ref_array<props>(*src, parent); + default: + throw cast_error("unhandled return_value_policy: should not happen!"); + }; + } + +public: + // Normal returned non-reference, non-const value: + static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // If you return a non-reference const, we mark the numpy array readonly: + static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // lvalue reference return; default (automatic) becomes copy + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + // const lvalue reference return; default (automatic) becomes copy + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + // non-const pointer return + static handle cast(Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + // const pointer return + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + + static constexpr auto name = props::descriptor; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type *() { return &value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &() { return value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &&() && { return std::move(value); } + template <typename T> + using cast_op_type = movable_cast_op_type<T>; + +private: + Type value; +}; + +// Base class for casting reference/map/block/etc. objects back to python. +template <typename MapType> +struct eigen_map_caster { + static_assert(!std::is_pointer<typename MapType::Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + +private: + using props = EigenProps<MapType>; + +public: + // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has + // to stay around), but we'll allow it under the assumption that you know what you're doing + // (and have an appropriate keep_alive in place). We return a numpy array pointing directly at + // the ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) + // Note that this means you need to ensure you don't destroy the object in some other way (e.g. + // with an appropriate keep_alive, or with a reference to a statically allocated matrix). + static handle cast(const MapType &src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::copy: + return eigen_array_cast<props>(src); + case return_value_policy::reference_internal: + return eigen_array_cast<props>(src, parent, is_eigen_mutable_map<MapType>::value); + case return_value_policy::reference: + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + return eigen_array_cast<props>(src, none(), is_eigen_mutable_map<MapType>::value); + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); + } + } + + static constexpr auto name = props::descriptor; + + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator MapType() = delete; + template <typename> + using cast_op_type = MapType; +}; + +// We can return any map-like object (but can only load Refs, specialized next): +template <typename Type> +struct type_caster<Type, enable_if_t<is_eigen_dense_map<Type>::value>> : eigen_map_caster<Type> {}; + +// Loader for Ref<...> arguments. See the documentation for info on how to make this work without +// copying (it requires some extra effort in many cases). +template <typename PlainObjectType, typename StrideType> +struct type_caster< + Eigen::Ref<PlainObjectType, 0, StrideType>, + enable_if_t<is_eigen_dense_map<Eigen::Ref<PlainObjectType, 0, StrideType>>::value>> + : public eigen_map_caster<Eigen::Ref<PlainObjectType, 0, StrideType>> { +private: + using Type = Eigen::Ref<PlainObjectType, 0, StrideType>; + using props = EigenProps<Type>; + using Scalar = typename props::Scalar; + static_assert(!std::is_pointer<Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using MapType = Eigen::Map<PlainObjectType, 0, StrideType>; + using Array + = array_t<Scalar, + array::forcecast + | ((props::row_major ? props::inner_stride : props::outer_stride) == 1 + ? array::c_style + : (props::row_major ? props::outer_stride : props::inner_stride) == 1 + ? array::f_style + : 0)>; + static constexpr bool need_writeable = is_eigen_mutable_map<Type>::value; + // Delay construction (these have no default constructor) + std::unique_ptr<MapType> map; + std::unique_ptr<Type> ref; + // Our array. When possible, this is just a numpy array pointing to the source data, but + // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an + // incompatible layout, or is an array of a type that needs to be converted). Using a numpy + // temporary (rather than an Eigen temporary) saves an extra copy when we need both type + // conversion and storage order conversion. (Note that we refuse to use this temporary copy + // when loading an argument for a Ref<M> with M non-const, i.e. a read-write reference). + Array copy_or_ref; + +public: + bool load(handle src, bool convert) { + // First check whether what we have is already an array of the right type. If not, we + // can't avoid a copy (because the copy is also going to do type conversion). + bool need_copy = !isinstance<Array>(src); + + EigenConformable<props::row_major> fits; + if (!need_copy) { + // We don't need a converting copy, but we also need to check whether the strides are + // compatible with the Ref's stride requirements + auto aref = reinterpret_borrow<Array>(src); + + if (aref && (!need_writeable || aref.writeable())) { + fits = props::conformable(aref); + if (!fits) { + return false; // Incompatible dimensions + } + if (!fits.template stride_compatible<props>()) { + need_copy = true; + } else { + copy_or_ref = std::move(aref); + } + } else { + need_copy = true; + } + } + + if (need_copy) { + // We need to copy: If we need a mutable reference, or we're not supposed to convert + // (either because we're in the no-convert overload pass, or because we're explicitly + // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. + if (!convert || need_writeable) { + return false; + } + + Array copy = Array::ensure(src); + if (!copy) { + return false; + } + fits = props::conformable(copy); + if (!fits || !fits.template stride_compatible<props>()) { + return false; + } + copy_or_ref = std::move(copy); + loader_life_support::add_patient(copy_or_ref); + } + + ref.reset(); + map.reset(new MapType(data(copy_or_ref), + fits.rows, + fits.cols, + make_stride(fits.stride.outer(), fits.stride.inner()))); + ref.reset(new Type(*map)); + + return true; + } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type *() { return ref.get(); } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &() { return *ref; } + template <typename _T> + using cast_op_type = pybind11::detail::cast_op_type<_T>; + +private: + template <typename T = Type, enable_if_t<is_eigen_mutable_map<T>::value, int> = 0> + Scalar *data(Array &a) { + return a.mutable_data(); + } + + template <typename T = Type, enable_if_t<!is_eigen_mutable_map<T>::value, int> = 0> + const Scalar *data(Array &a) { + return a.data(); + } + + // Attempt to figure out a constructor of `Stride` that will work. + // If both strides are fixed, use a default constructor: + template <typename S> + using stride_ctor_default = bool_constant<S::InnerStrideAtCompileTime != Eigen::Dynamic + && S::OuterStrideAtCompileTime != Eigen::Dynamic + && std::is_default_constructible<S>::value>; + // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like + // Eigen::Stride, and use it: + template <typename S> + using stride_ctor_dual + = bool_constant<!stride_ctor_default<S>::value + && std::is_constructible<S, EigenIndex, EigenIndex>::value>; + // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use + // it (passing whichever stride is dynamic). + template <typename S> + using stride_ctor_outer + = bool_constant<!any_of<stride_ctor_default<S>, stride_ctor_dual<S>>::value + && S::OuterStrideAtCompileTime == Eigen::Dynamic + && S::InnerStrideAtCompileTime != Eigen::Dynamic + && std::is_constructible<S, EigenIndex>::value>; + template <typename S> + using stride_ctor_inner + = bool_constant<!any_of<stride_ctor_default<S>, stride_ctor_dual<S>>::value + && S::InnerStrideAtCompileTime == Eigen::Dynamic + && S::OuterStrideAtCompileTime != Eigen::Dynamic + && std::is_constructible<S, EigenIndex>::value>; + + template <typename S = StrideType, enable_if_t<stride_ctor_default<S>::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex) { + return S(); + } + template <typename S = StrideType, enable_if_t<stride_ctor_dual<S>::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex inner) { + return S(outer, inner); + } + template <typename S = StrideType, enable_if_t<stride_ctor_outer<S>::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex) { + return S(outer); + } + template <typename S = StrideType, enable_if_t<stride_ctor_inner<S>::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex inner) { + return S(inner); + } +}; + +// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not +// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). +// load() is not supported, but we can cast them into the python domain by first copying to a +// regular Eigen::Matrix, then casting that. +template <typename Type> +struct type_caster<Type, enable_if_t<is_eigen_other<Type>::value>> { + static_assert(!std::is_pointer<typename Type::Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + +protected: + using Matrix + = Eigen::Matrix<typename Type::Scalar, Type::RowsAtCompileTime, Type::ColsAtCompileTime>; + using props = EigenProps<Matrix>; + +public: + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + handle h = eigen_encapsulate<props>(new Matrix(src)); + return h; + } + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast(*src, policy, parent); + } + + static constexpr auto name = props::descriptor; + + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator Type() = delete; + template <typename> + using cast_op_type = Type; +}; + +template <typename Type> +struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> { + using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer<Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using StorageIndex = remove_reference_t<decltype(*std::declval<Type>().outerIndexPtr())>; + using Index = typename Type::Index; + static constexpr bool rowMajor = Type::IsRowMajor; + + bool load(handle src, bool) { + if (!src) { + return false; + } + + auto obj = reinterpret_borrow<object>(src); + object sparse_module = module_::import("scipy.sparse"); + object matrix_type = sparse_module.attr(rowMajor ? "csr_matrix" : "csc_matrix"); + + if (!type::handle_of(obj).is(matrix_type)) { + try { + obj = matrix_type(obj); + } catch (const error_already_set &) { + return false; + } + } + + auto values = array_t<Scalar>((object) obj.attr("data")); + auto innerIndices = array_t<StorageIndex>((object) obj.attr("indices")); + auto outerIndices = array_t<StorageIndex>((object) obj.attr("indptr")); + auto shape = pybind11::tuple((pybind11::object) obj.attr("shape")); + auto nnz = obj.attr("nnz").cast<Index>(); + + if (!values || !innerIndices || !outerIndices) { + return false; + } + + value = EigenMapSparseMatrix<Scalar, + Type::Flags &(Eigen::RowMajor | Eigen::ColMajor), + StorageIndex>(shape[0].cast<Index>(), + shape[1].cast<Index>(), + std::move(nnz), + outerIndices.mutable_data(), + innerIndices.mutable_data(), + values.mutable_data()); + + return true; + } + + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + const_cast<Type &>(src).makeCompressed(); + + object matrix_type + = module_::import("scipy.sparse").attr(rowMajor ? "csr_matrix" : "csc_matrix"); + + array data(src.nonZeros(), src.valuePtr()); + array outerIndices((rowMajor ? src.rows() : src.cols()) + 1, src.outerIndexPtr()); + array innerIndices(src.nonZeros(), src.innerIndexPtr()); + + return matrix_type(pybind11::make_tuple( + std::move(data), std::move(innerIndices), std::move(outerIndices)), + pybind11::make_tuple(src.rows(), src.cols())) + .release(); + } + + PYBIND11_TYPE_CASTER(Type, + const_name<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", + "scipy.sparse.csc_matrix[") + + npy_format_descriptor<Scalar>::name + const_name("]")); +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h new file mode 100644 index 00000000..25d12bac --- /dev/null +++ b/include/pybind11/eigen/tensor.h @@ -0,0 +1,516 @@ +/* + pybind11/eigen/tensor.h: Transparent conversion for Eigen tensors + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../numpy.h" +#include "common.h" + +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) +static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); +#endif + +// Disable warnings for Eigen +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4554) +PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif + +#include <unsupported/Eigen/CXX11/Tensor> + +PYBIND11_WARNING_POP + +static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), + "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +PYBIND11_NAMESPACE_BEGIN(detail) + +inline bool is_tensor_aligned(const void *data) { + return (reinterpret_cast<std::size_t>(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; +} + +template <typename T> +constexpr int compute_array_flag_from_tensor() { + static_assert((static_cast<int>(T::Layout) == static_cast<int>(Eigen::RowMajor)) + || (static_cast<int>(T::Layout) == static_cast<int>(Eigen::ColMajor)), + "Layout must be row or column major"); + return (static_cast<int>(T::Layout) == static_cast<int>(Eigen::RowMajor)) ? array::c_style + : array::f_style; +} + +template <typename T> +struct eigen_tensor_helper {}; + +template <typename Scalar_, int NumIndices_, int Options_, typename IndexType> +struct eigen_tensor_helper<Eigen::Tensor<Scalar_, NumIndices_, Options_, IndexType>> { + using Type = Eigen::Tensor<Scalar_, NumIndices_, Options_, IndexType>; + using ValidType = void; + + static Eigen::DSizes<typename Type::Index, Type::NumIndices> get_shape(const Type &f) { + return f.dimensions(); + } + + static constexpr bool + is_correct_shape(const Eigen::DSizes<typename Type::Index, Type::NumIndices> & /*shape*/) { + return true; + } + + template <typename T> + struct helper {}; + + template <size_t... Is> + struct helper<index_sequence<Is...>> { + static constexpr auto value = concat(const_name(((void) Is, "?"))...); + }; + + static constexpr auto dimensions_descriptor + = helper<decltype(make_index_sequence<Type::NumIndices>())>::value; + + template <typename... Args> + static Type *alloc(Args &&...args) { + return new Type(std::forward<Args>(args)...); + } + + static void free(Type *tensor) { delete tensor; } +}; + +template <typename Scalar_, typename std::ptrdiff_t... Indices, int Options_, typename IndexType> +struct eigen_tensor_helper< + Eigen::TensorFixedSize<Scalar_, Eigen::Sizes<Indices...>, Options_, IndexType>> { + using Type = Eigen::TensorFixedSize<Scalar_, Eigen::Sizes<Indices...>, Options_, IndexType>; + using ValidType = void; + + static constexpr Eigen::DSizes<typename Type::Index, Type::NumIndices> + get_shape(const Type & /*f*/) { + return get_shape(); + } + + static constexpr Eigen::DSizes<typename Type::Index, Type::NumIndices> get_shape() { + return Eigen::DSizes<typename Type::Index, Type::NumIndices>(Indices...); + } + + static bool + is_correct_shape(const Eigen::DSizes<typename Type::Index, Type::NumIndices> &shape) { + return get_shape() == shape; + } + + static constexpr auto dimensions_descriptor = concat(const_name<Indices>()...); + + template <typename... Args> + static Type *alloc(Args &&...args) { + Eigen::aligned_allocator<Type> allocator; + return ::new (allocator.allocate(1)) Type(std::forward<Args>(args)...); + } + + static void free(Type *tensor) { + Eigen::aligned_allocator<Type> allocator; + tensor->~Type(); + allocator.deallocate(tensor, 1); + } +}; + +template <typename Type, bool ShowDetails, bool NeedsWriteable = false> +struct get_tensor_descriptor { + static constexpr auto details + = const_name<NeedsWriteable>(", flags.writeable", "") + + const_name<static_cast<int>(Type::Layout) == static_cast<int>(Eigen::RowMajor)>( + ", flags.c_contiguous", ", flags.f_contiguous"); + static constexpr auto value + = const_name("numpy.ndarray[") + npy_format_descriptor<typename Type::Scalar>::name + + const_name("[") + eigen_tensor_helper<remove_cv_t<Type>>::dimensions_descriptor + + const_name("]") + const_name<ShowDetails>(details, const_name("")) + const_name("]"); +}; + +// When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes<T, 0> does not have the begin() member +// function. Falling back to a simple loop works around this issue. +// +// We need to disable the type-limits warning for the inner loop when size = 0. + +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wtype-limits") + +template <typename T, int size> +std::vector<T> convert_dsizes_to_vector(const Eigen::DSizes<T, size> &arr) { + std::vector<T> result(size); + + for (size_t i = 0; i < size; i++) { + result[i] = arr[i]; + } + + return result; +} + +template <typename T, int size> +Eigen::DSizes<T, size> get_shape_for_array(const array &arr) { + Eigen::DSizes<T, size> result; + const T *shape = arr.shape(); + for (size_t i = 0; i < size; i++) { + result[i] = shape[i]; + } + + return result; +} + +PYBIND11_WARNING_POP + +template <typename Type> +struct type_caster<Type, typename eigen_tensor_helper<Type>::ValidType> { + static_assert(!std::is_pointer<typename Type::Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using Helper = eigen_tensor_helper<Type>; + static constexpr auto temp_name = get_tensor_descriptor<Type, false>::value; + PYBIND11_TYPE_CASTER(Type, temp_name); + + bool load(handle src, bool convert) { + if (!convert) { + if (!isinstance<array>(src)) { + return false; + } + array temp = array::ensure(src); + if (!temp) { + return false; + } + + if (!temp.dtype().is(dtype::of<typename Type::Scalar>())) { + return false; + } + } + + array_t<typename Type::Scalar, compute_array_flag_from_tensor<Type>()> arr( + reinterpret_borrow<object>(src)); + + if (arr.ndim() != Type::NumIndices) { + return false; + } + auto shape = get_shape_for_array<typename Type::Index, Type::NumIndices>(arr); + + if (!Helper::is_correct_shape(shape)) { + return false; + } + +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + auto data_pointer = arr.data(); +#else + // Handle Eigen bug + auto data_pointer = const_cast<typename Type::Scalar *>(arr.data()); +#endif + + if (is_tensor_aligned(arr.data())) { + value = Eigen::TensorMap<const Type, Eigen::Aligned>(data_pointer, shape); + } else { + value = Eigen::TensorMap<const Type>(data_pointer, shape); + } + + return true; + } + + static handle cast(Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(const Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template <typename C> + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + object parent_object; + bool writeable = false; + switch (policy) { + case return_value_policy::move: + if (std::is_const<C>::value) { + pybind11_fail("Cannot move from a constant reference"); + } + + src = Helper::alloc(std::move(*src)); + + parent_object + = capsule(src, [](void *ptr) { Helper::free(reinterpret_cast<Type *>(ptr)); }); + writeable = true; + break; + + case return_value_policy::take_ownership: + if (std::is_const<C>::value) { + // This cast is ugly, and might be UB in some cases, but we don't have an + // alternative here as we must free that memory + Helper::free(const_cast<Type *>(src)); + pybind11_fail("Cannot take ownership of a const reference"); + } + + parent_object + = capsule(src, [](void *ptr) { Helper::free(reinterpret_cast<Type *>(ptr)); }); + writeable = true; + break; + + case return_value_policy::copy: + writeable = true; + break; + + case return_value_policy::reference: + parent_object = none(); + writeable = !std::is_const<C>::value; + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + if (!parent) { + pybind11_fail("Cannot use reference internal when there is no parent"); + } + parent_object = reinterpret_borrow<object>(parent); + writeable = !std::is_const<C>::value; + break; + + default: + pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); + } + + auto result = array_t<typename Type::Scalar, compute_array_flag_from_tensor<Type>()>( + convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), parent_object); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result.release(); + } +}; + +template <typename StoragePointerType, + bool needs_writeable, + enable_if_t<!needs_writeable, bool> = true> +StoragePointerType get_array_data_for_type(array &arr) { +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + return reinterpret_cast<StoragePointerType>(arr.data()); +#else + // Handle Eigen bug + return reinterpret_cast<StoragePointerType>(const_cast<void *>(arr.data())); +#endif +} + +template <typename StoragePointerType, + bool needs_writeable, + enable_if_t<needs_writeable, bool> = true> +StoragePointerType get_array_data_for_type(array &arr) { + return reinterpret_cast<StoragePointerType>(arr.mutable_data()); +} + +template <typename T, typename = void> +struct get_storage_pointer_type; + +template <typename MapType> +struct get_storage_pointer_type<MapType, void_t<typename MapType::StoragePointerType>> { + using SPT = typename MapType::StoragePointerType; +}; + +template <typename MapType> +struct get_storage_pointer_type<MapType, void_t<typename MapType::PointerArgType>> { + using SPT = typename MapType::PointerArgType; +}; + +template <typename Type, int Options> +struct type_caster<Eigen::TensorMap<Type, Options>, + typename eigen_tensor_helper<remove_cv_t<Type>>::ValidType> { + static_assert(!std::is_pointer<typename Type::Scalar>::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using MapType = Eigen::TensorMap<Type, Options>; + using Helper = eigen_tensor_helper<remove_cv_t<Type>>; + + bool load(handle src, bool /*convert*/) { + // Note that we have a lot more checks here as we want to make sure to avoid copies + if (!isinstance<array>(src)) { + return false; + } + auto arr = reinterpret_borrow<array>(src); + if ((arr.flags() & compute_array_flag_from_tensor<Type>()) == 0) { + return false; + } + + if (!arr.dtype().is(dtype::of<typename Type::Scalar>())) { + return false; + } + + if (arr.ndim() != Type::NumIndices) { + return false; + } + + constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; + + if (is_aligned && !is_tensor_aligned(arr.data())) { + return false; + } + + auto shape = get_shape_for_array<typename Type::Index, Type::NumIndices>(arr); + + if (!Helper::is_correct_shape(shape)) { + return false; + } + + if (needs_writeable && !arr.writeable()) { + return false; + } + + auto result = get_array_data_for_type<typename get_storage_pointer_type<MapType>::SPT, + needs_writeable>(arr); + + value.reset(new MapType(std::move(result), std::move(shape))); + + return true; + } + + static handle cast(MapType &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(const MapType &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(MapType &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const MapType &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(MapType *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const MapType *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template <typename C> + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + object parent_object; + constexpr bool writeable = !std::is_const<C>::value; + switch (policy) { + case return_value_policy::reference: + parent_object = none(); + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + if (!parent) { + pybind11_fail("Cannot use reference internal when there is no parent"); + } + parent_object = reinterpret_borrow<object>(parent); + break; + + case return_value_policy::take_ownership: + delete src; + // fallthrough + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " + "reference or reference_internal"); + } + + auto result = array_t<typename Type::Scalar, compute_array_flag_from_tensor<Type>()>( + convert_dsizes_to_vector(Helper::get_shape(*src)), + src->data(), + std::move(parent_object)); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result.release(); + } + +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + + static constexpr bool needs_writeable = !std::is_const<typename std::remove_pointer< + typename get_storage_pointer_type<MapType>::SPT>::type>::value; +#else + // Handle Eigen bug + static constexpr bool needs_writeable = !std::is_const<Type>::value; +#endif + +protected: + // TODO: Move to std::optional once std::optional has more support + std::unique_ptr<MapType> value; + +public: + static constexpr auto name = get_tensor_descriptor<Type, true, needs_writeable>::value; + explicit operator MapType *() { return value.get(); } + explicit operator MapType &() { return *value; } + explicit operator MapType &&() && { return std::move(*value); } + + template <typename T_> + using cast_op_type = ::pybind11::detail::movable_cast_op_type<T_>; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 204aaf98..caa14f4a 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -12,23 +12,16 @@ #include "pybind11.h" #include "eval.h" +#include <memory> +#include <vector> + #if defined(PYPY_VERSION) -# error Embedding the interpreter is not supported with PyPy +# error Embedding the interpreter is not supported with PyPy #endif -#if PY_MAJOR_VERSION >= 3 -# define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ - extern "C" PyObject *pybind11_init_impl_##name(); \ - extern "C" PyObject *pybind11_init_impl_##name() { \ - return pybind11_init_wrapper_##name(); \ - } -#else -# define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ - extern "C" void pybind11_init_impl_##name(); \ - extern "C" void pybind11_init_impl_##name() { \ - pybind11_init_wrapper_##name(); \ - } -#endif +#define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + extern "C" PyObject *pybind11_init_impl_##name(); \ + extern "C" PyObject *pybind11_init_impl_##name() { return pybind11_init_wrapper_##name(); } /** \rst Add a new module to the table of builtins for the interpreter. Must be @@ -45,69 +38,173 @@ }); } \endrst */ -#define PYBIND11_EMBEDDED_MODULE(name, variable) \ - static ::pybind11::module_::module_def \ - PYBIND11_CONCAT(pybind11_module_def_, name); \ - static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ - static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \ - auto m = ::pybind11::module_::create_extension_module( \ - PYBIND11_TOSTRING(name), nullptr, \ - &PYBIND11_CONCAT(pybind11_module_def_, name)); \ - try { \ - PYBIND11_CONCAT(pybind11_init_, name)(m); \ - return m.ptr(); \ - } PYBIND11_CATCH_INIT_EXCEPTIONS \ - } \ - PYBIND11_EMBEDDED_MODULE_IMPL(name) \ - ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name) \ - (PYBIND11_TOSTRING(name), \ - PYBIND11_CONCAT(pybind11_init_impl_, name)); \ - void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &variable) - +#define PYBIND11_EMBEDDED_MODULE(name, variable) \ + static ::pybind11::module_::module_def PYBIND11_CONCAT(pybind11_module_def_, name); \ + static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ + static PyObject PYBIND11_CONCAT(*pybind11_init_wrapper_, name)() { \ + auto m = ::pybind11::module_::create_extension_module( \ + PYBIND11_TOSTRING(name), nullptr, &PYBIND11_CONCAT(pybind11_module_def_, name)); \ + try { \ + PYBIND11_CONCAT(pybind11_init_, name)(m); \ + return m.ptr(); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + } \ + PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + ::pybind11::detail::embedded_module PYBIND11_CONCAT(pybind11_module_, name)( \ + PYBIND11_TOSTRING(name), PYBIND11_CONCAT(pybind11_init_impl_, name)); \ + void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ \ + & variable) // NOLINT(bugprone-macro-parentheses) PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) /// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks. struct embedded_module { -#if PY_MAJOR_VERSION >= 3 - using init_t = PyObject *(*)(); -#else - using init_t = void (*)(); -#endif + using init_t = PyObject *(*) (); embedded_module(const char *name, init_t init) { - if (Py_IsInitialized()) + if (Py_IsInitialized() != 0) { pybind11_fail("Can't add new modules after the interpreter has been initialized"); + } auto result = PyImport_AppendInittab(name, init); - if (result == -1) + if (result == -1) { pybind11_fail("Insufficient memory to add a new module"); + } + } +}; + +struct wide_char_arg_deleter { + void operator()(wchar_t *ptr) const { + // API docs: https://docs.python.org/3/c-api/sys.html#c.Py_DecodeLocale + PyMem_RawFree(ptr); } }; +inline wchar_t *widen_chars(const char *safe_arg) { + wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr); + return widened_arg; +} + +inline void precheck_interpreter() { + if (Py_IsInitialized() != 0) { + pybind11_fail("The interpreter is already running"); + } +} + +#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX) +# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000) +#endif + +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, + int argc, + const char *const *argv, + bool add_program_dir_to_path) { + detail::precheck_interpreter(); + Py_InitializeEx(init_signal_handlers ? 1 : 0); +# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +# endif + + // Before it was special-cased in python 3.8, passing an empty or null argv + // caused a segfault, so we have to reimplement the special case ourselves. + bool special_case = (argv == nullptr || argc <= 0); + + const char *const empty_argv[]{"\0"}; + const char *const *safe_argv = special_case ? empty_argv : argv; + if (special_case) { + argc = 1; + } + + auto argv_size = static_cast<size_t>(argc); + // SetArgv* on python 3 takes wchar_t, so we have to convert. + std::unique_ptr<wchar_t *[]> widened_argv(new wchar_t *[argv_size]); + std::vector<std::unique_ptr<wchar_t[], detail::wide_char_arg_deleter>> widened_argv_entries; + widened_argv_entries.reserve(argv_size); + for (size_t ii = 0; ii < argv_size; ++ii) { + widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii])); + if (!widened_argv_entries.back()) { + // A null here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + return; + } + widened_argv[ii] = widened_argv_entries.back().get(); + } + + auto *pysys_argv = widened_argv.get(); + + PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path)); +} +#endif + PYBIND11_NAMESPACE_END(detail) +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + detail::precheck_interpreter(); + PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv)); + if (PyStatus_Exception(status) != 0) { + // A failure here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to prepare CPython"); + } + status = Py_InitializeFromConfig(config); + if (PyStatus_Exception(status) != 0) { + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to init CPython"); + } + if (add_program_dir_to_path) { + PyRun_SimpleString("import sys, os.path; " + "sys.path.insert(0, " + "os.path.abspath(os.path.dirname(sys.argv[0])) " + "if sys.argv and os.path.exists(sys.argv[0]) else '')"); + } + PyConfig_Clear(config); +} +#endif + /** \rst Initialize the Python interpreter. No other pybind11 or CPython API functions can be called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The - optional parameter can be used to skip the registration of signal handlers (see the - `Python documentation`_ for details). Calling this function again after the interpreter - has already been initialized is a fatal error. + optional `init_signal_handlers` parameter can be used to skip the registration of + signal handlers (see the `Python documentation`_ for details). Calling this function + again after the interpreter has already been initialized is a fatal error. If initializing the Python interpreter fails, then the program is terminated. (This is controlled by the CPython runtime and is an exception to pybind11's normal behavior of throwing exceptions on errors.) + The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are + used to populate ``sys.argv`` and ``sys.path``. + See the |PySys_SetArgvEx documentation|_ for details. + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx + .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation + .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx \endrst */ -inline void initialize_interpreter(bool init_signal_handlers = true) { - if (Py_IsInitialized()) - pybind11_fail("The interpreter is already running"); - - Py_InitializeEx(init_signal_handlers ? 1 : 0); +inline void initialize_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + detail::initialize_interpreter_pre_pyconfig( + init_signal_handlers, argc, argv, add_program_dir_to_path); +#else + PyConfig config; + PyConfig_InitPythonConfig(&config); + // See PR #4473 for background + config.parse_argv = 0; - // Make .py files in the working directory available by default - module_::import("sys").attr("path").cast<list>().append("."); + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + initialize_interpreter(&config, argc, argv, add_program_dir_to_path); +#endif } /** \rst @@ -146,16 +243,19 @@ inline void initialize_interpreter(bool init_signal_handlers = true) { \endrst */ inline void finalize_interpreter() { - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - // Get the internals pointer (without creating it if it doesn't exist). It's possible for the // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()` // during destruction), so we get the pointer-pointer here and check it after Py_Finalize(). detail::internals **internals_ptr_ptr = detail::get_internals_pp(); - // It could also be stashed in builtins, so look there too: - if (builtins.contains(id) && isinstance<capsule>(builtins[id])) - internals_ptr_ptr = capsule(builtins[id]); + // It could also be stashed in state_dict, so look there too: + if (object internals_obj + = get_internals_obj_from_state_dict(detail::get_python_state_dict())) { + internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj); + } + // Local internals contains data managed by the current interpreter, so we must clear them to + // avoid undefined behaviors when initializing another interpreter + detail::get_local_internals().registered_types_cpp.clear(); + detail::get_local_internals().registered_exception_translators.clear(); Py_Finalize(); @@ -169,6 +269,8 @@ inline void finalize_interpreter() { Scope guard version of `initialize_interpreter` and `finalize_interpreter`. This a move-only guard and only a single instance can exist. + See `initialize_interpreter` for a discussion of its constructor arguments. + .. code-block:: cpp #include <pybind11/embed.h> @@ -180,9 +282,21 @@ inline void finalize_interpreter() { \endrst */ class scoped_interpreter { public: - scoped_interpreter(bool init_signal_handlers = true) { - initialize_interpreter(init_signal_handlers); + explicit scoped_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); + } + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + explicit scoped_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(config, argc, argv, add_program_dir_to_path); } +#endif scoped_interpreter(const scoped_interpreter &) = delete; scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } @@ -190,8 +304,9 @@ public: scoped_interpreter &operator=(scoped_interpreter &&) = delete; ~scoped_interpreter() { - if (is_valid) + if (is_valid) { finalize_interpreter(); + } } private: diff --git a/include/pybind11/eval.h b/include/pybind11/eval.h index fa6b8af4..bd5f981f 100644 --- a/include/pybind11/eval.h +++ b/include/pybind11/eval.h @@ -1,5 +1,5 @@ /* - pybind11/exec.h: Support for evaluating Python expressions and statements + pybind11/eval.h: Support for evaluating Python expressions and statements from strings and files Copyright (c) 2016 Klemens Morgenstern <klemens.morgenstern@ed-chemnitz.de> and @@ -13,20 +13,22 @@ #include "pybind11.h" +#include <utility> + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline void ensure_builtins_in_globals(object &global) { - #if PY_VERSION_HEX < 0x03080000 - // Running exec and eval on Python 2 and 3 adds `builtins` module under - // `__builtins__` key to globals if not yet present. - // Python 3.8 made PyRun_String behave similarly. Let's also do that for - // older versions, for consistency. - if (!global.contains("__builtins__")) - global["__builtins__"] = module_::import(PYBIND11_BUILTINS_MODULE); - #else - (void) global; - #endif +#if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x03080000 + // Running exec and eval adds `builtins` module under `__builtins__` key to + // globals if not yet present. Python 3.8 made PyRun_String behave + // similarly. Let's also do that for older versions, for consistency. This + // was missing from PyPy3.8 7.3.7. + if (!global.contains("__builtins__")) + global["__builtins__"] = module_::import(PYBIND11_BUILTINS_MODULE); +#else + (void) global; +#endif } PYBIND11_NAMESPACE_END(detail) @@ -43,9 +45,10 @@ enum eval_mode { }; template <eval_mode mode = eval_expr> -object eval(str expr, object global = globals(), object local = object()) { - if (!local) +object eval(const str &expr, object global = globals(), object local = object()) { + if (!local) { local = global; + } detail::ensure_builtins_in_globals(global); @@ -53,38 +56,45 @@ object eval(str expr, object global = globals(), object local = object()) { this seems to be the only alternative */ std::string buffer = "# -*- coding: utf-8 -*-\n" + (std::string) expr; - int start; + int start = 0; switch (mode) { - case eval_expr: start = Py_eval_input; break; - case eval_single_statement: start = Py_single_input; break; - case eval_statements: start = Py_file_input; break; - default: pybind11_fail("invalid evaluation mode"); + case eval_expr: + start = Py_eval_input; + break; + case eval_single_statement: + start = Py_single_input; + break; + case eval_statements: + start = Py_file_input; + break; + default: + pybind11_fail("invalid evaluation mode"); } PyObject *result = PyRun_String(buffer.c_str(), start, global.ptr(), local.ptr()); - if (!result) + if (!result) { throw error_already_set(); + } return reinterpret_steal<object>(result); } template <eval_mode mode = eval_expr, size_t N> object eval(const char (&s)[N], object global = globals(), object local = object()) { /* Support raw string literals by removing common leading whitespace */ - auto expr = (s[0] == '\n') ? str(module_::import("textwrap").attr("dedent")(s)) - : str(s); - return eval<mode>(expr, global, local); + auto expr = (s[0] == '\n') ? str(module_::import("textwrap").attr("dedent")(s)) : str(s); + return eval<mode>(expr, std::move(global), std::move(local)); } -inline void exec(str expr, object global = globals(), object local = object()) { - eval<eval_statements>(expr, global, local); +inline void exec(const str &expr, object global = globals(), object local = object()) { + eval<eval_statements>(expr, std::move(global), std::move(local)); } template <size_t N> void exec(const char (&s)[N], object global = globals(), object local = object()) { - eval<eval_statements>(s, global, local); + eval<eval_statements>(s, std::move(global), std::move(local)); } -#if defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x03000000 +#if defined(PYPY_VERSION) template <eval_mode mode = eval_statements> object eval_file(str, object, object) { pybind11_fail("eval_file not supported in PyPy3. Use eval"); @@ -100,51 +110,45 @@ object eval_file(str) { #else template <eval_mode mode = eval_statements> object eval_file(str fname, object global = globals(), object local = object()) { - if (!local) + if (!local) { local = global; + } detail::ensure_builtins_in_globals(global); - int start; + int start = 0; switch (mode) { - case eval_expr: start = Py_eval_input; break; - case eval_single_statement: start = Py_single_input; break; - case eval_statements: start = Py_file_input; break; - default: pybind11_fail("invalid evaluation mode"); + case eval_expr: + start = Py_eval_input; + break; + case eval_single_statement: + start = Py_single_input; + break; + case eval_statements: + start = Py_file_input; + break; + default: + pybind11_fail("invalid evaluation mode"); } int closeFile = 1; std::string fname_str = (std::string) fname; -#if PY_VERSION_HEX >= 0x03040000 FILE *f = _Py_fopen_obj(fname.ptr(), "r"); -#elif PY_VERSION_HEX >= 0x03000000 - FILE *f = _Py_fopen(fname.ptr(), "r"); -#else - /* No unicode support in open() :( */ - auto fobj = reinterpret_steal<object>(PyFile_FromString( - const_cast<char *>(fname_str.c_str()), - const_cast<char*>("r"))); - FILE *f = nullptr; - if (fobj) - f = PyFile_AsFile(fobj.ptr()); - closeFile = 0; -#endif if (!f) { PyErr_Clear(); pybind11_fail("File \"" + fname_str + "\" could not be opened!"); } -#if PY_VERSION_HEX < 0x03000000 && defined(PYPY_VERSION) - PyObject *result = PyRun_File(f, fname_str.c_str(), start, global.ptr(), - local.ptr()); - (void) closeFile; -#else - PyObject *result = PyRun_FileEx(f, fname_str.c_str(), start, global.ptr(), - local.ptr(), closeFile); -#endif + if (!global.contains("__file__")) { + global["__file__"] = std::move(fname); + } + + PyObject *result + = PyRun_FileEx(f, fname_str.c_str(), start, global.ptr(), local.ptr(), closeFile); - if (!result) + if (!result) { throw error_already_set(); + } return reinterpret_steal<object>(result); } #endif diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 92c17dc2..87ec4d10 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -10,6 +10,7 @@ #pragma once #include "pybind11.h" + #include <functional> PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -19,18 +20,21 @@ template <typename Return, typename... Args> struct type_caster<std::function<Return(Args...)>> { using type = std::function<Return(Args...)>; using retval_type = conditional_t<std::is_same<Return, void>::value, void_type, Return>; - using function_type = Return (*) (Args...); + using function_type = Return (*)(Args...); public: bool load(handle src, bool convert) { if (src.is_none()) { // Defer accepting None to other overloads (if we aren't in convert mode): - if (!convert) return false; + if (!convert) { + return false; + } return true; } - if (!isinstance<function>(src)) + if (!isinstance<function>(src)) { return false; + } auto func = reinterpret_borrow<function>(src); @@ -43,24 +47,51 @@ public: captured variables), in which case the roundtrip can be avoided. */ if (auto cfunc = func.cpp_function()) { - auto c = reinterpret_borrow<capsule>(PyCFunction_GET_SELF(cfunc.ptr())); - auto rec = (function_record *) c; - - if (rec && rec->is_stateless && - same_type(typeid(function_type), *reinterpret_cast<const std::type_info *>(rec->data[1]))) { - struct capture { function_type f; }; - value = ((capture *) &rec->data)->f; - return true; + auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); + if (cfunc_self == nullptr) { + PyErr_Clear(); + } else if (isinstance<capsule>(cfunc_self)) { + auto c = reinterpret_borrow<capsule>(cfunc_self); + + function_record *rec = nullptr; + // Check that we can safely reinterpret the capsule into a function_record + if (detail::is_function_record_capsule(c)) { + rec = c.get_pointer<function_record>(); + } + + while (rec != nullptr) { + if (rec->is_stateless + && same_type(typeid(function_type), + *reinterpret_cast<const std::type_info *>(rec->data[1]))) { + struct capture { + function_type f; + }; + value = ((capture *) &rec->data)->f; + return true; + } + rec = rec->next; + } } + // PYPY segfaults here when passing builtin function like sum. + // Raising an fail exception here works to prevent the segfault, but only on gcc. + // See PR #1413 for full details } // ensure GIL is held during functor destruction struct func_handle { function f; - func_handle(function&& f_) : f(std::move(f_)) {} - func_handle(const func_handle& f_) { +#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17)) + // This triggers a syntax error under very special conditions (very weird indeed). + explicit +#endif + func_handle(function &&f_) noexcept + : f(std::move(f_)) { + } + func_handle(const func_handle &f_) { operator=(f_); } + func_handle &operator=(const func_handle &f_) { gil_scoped_acquire acq; f = f_.f; + return *this; } ~func_handle() { gil_scoped_acquire acq; @@ -71,12 +102,11 @@ public: // to emulate 'move initialization capture' in C++11 struct func_wrapper { func_handle hfunc; - func_wrapper(func_handle&& hf): hfunc(std::move(hf)) {} + explicit func_wrapper(func_handle &&hf) noexcept : hfunc(std::move(hf)) {} Return operator()(Args... args) const { gil_scoped_acquire acq; - object retval(hfunc.f(std::forward<Args>(args)...)); - /* Visual studio 2015 parser issue: need parentheses around this expression */ - return (retval.template cast<Return>()); + // casts the returned object as a rvalue to the return type + return hfunc.f(std::forward<Args>(args)...).template cast<Return>(); } }; @@ -86,18 +116,21 @@ public: template <typename Func> static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) { - if (!f_) - return none().inc_ref(); + if (!f_) { + return none().release(); + } auto result = f_.template target<function_type>(); - if (result) + if (result) { return cpp_function(*result, policy).release(); - else - return cpp_function(std::forward<Func>(f_), policy).release(); + } + return cpp_function(std::forward<Func>(f_), policy).release(); } - PYBIND11_TYPE_CASTER(type, _("Callable[[") + concat(make_caster<Args>::name...) + _("], ") - + make_caster<retval_type>::name + _("]")); + PYBIND11_TYPE_CASTER(type, + const_name("Callable[[") + concat(make_caster<Args>::name...) + + const_name("], ") + make_caster<retval_type>::name + + const_name("]")); }; PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h new file mode 100644 index 00000000..570a5581 --- /dev/null +++ b/include/pybind11/gil.h @@ -0,0 +1,239 @@ +/* + pybind11/gil.h: RAII helpers for managing the GIL + + Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch> + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" + +#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/internals.h" +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_NAMESPACE_BEGIN(detail) + +// forward declarations +PyThreadState *get_thread_state_unchecked(); + +PYBIND11_NAMESPACE_END(detail) + +#if defined(WITH_THREAD) + +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + +/* The functions below essentially reproduce the PyGILState_* API using a RAII + * pattern, but there are a few important differences: + * + * 1. When acquiring the GIL from an non-main thread during the finalization + * phase, the GILState API blindly terminates the calling thread, which + * is often not what is wanted. This API does not do this. + * + * 2. The gil_scoped_release function can optionally cut the relationship + * of a PyThreadState and its associated thread, which allows moving it to + * another thread (this is a fairly rare/advanced use case). + * + * 3. The reference count of an acquired thread state can be controlled. This + * can be handy to prevent cases where callbacks issued from an external + * thread would otherwise constantly construct and destroy thread state data + * structures. + * + * See the Python bindings of NanoGUI (http://github.com/wjakob/nanogui) for an + * example which uses features 2 and 3 to migrate the Python thread of + * execution to another thread (to run the event loop on the original thread, + * in this case). + */ + +class gil_scoped_acquire { +public: + PYBIND11_NOINLINE gil_scoped_acquire() { + auto &internals = detail::get_internals(); + tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate); + + if (!tstate) { + /* Check if the GIL was acquired using the PyGILState_* API instead (e.g. if + calling from a Python thread). Since we use a different key, this ensures + we don't create a new thread state and deadlock in PyEval_AcquireThread + below. Note we don't save this state with internals.tstate, since we don't + create it we would fail to clear it (its reference count should be > 0). */ + tstate = PyGILState_GetThisThreadState(); + } + + if (!tstate) { + tstate = PyThreadState_New(internals.istate); +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (!tstate) { + pybind11_fail("scoped_acquire: could not create thread state!"); + } +# endif + tstate->gilstate_counter = 0; + PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); + } else { + release = detail::get_thread_state_unchecked() != tstate; + } + + if (release) { + PyEval_AcquireThread(tstate); + } + + inc_ref(); + } + + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; + + void inc_ref() { ++tstate->gilstate_counter; } + + PYBIND11_NOINLINE void dec_ref() { + --tstate->gilstate_counter; +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (detail::get_thread_state_unchecked() != tstate) { + pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); + } + if (tstate->gilstate_counter < 0) { + pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); + } +# endif + if (tstate->gilstate_counter == 0) { +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (!release) { + pybind11_fail("scoped_acquire::dec_ref(): internal error!"); + } +# endif + PyThreadState_Clear(tstate); + if (active) { + PyThreadState_DeleteCurrent(); + } + PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); + release = false; + } + } + + /// This method will disable the PyThreadState_DeleteCurrent call and the + /// GIL won't be acquired. This method should be used if the interpreter + /// could be shutting down when this is called, as thread deletion is not + /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and + /// protect subsequent code. + PYBIND11_NOINLINE void disarm() { active = false; } + + PYBIND11_NOINLINE ~gil_scoped_acquire() { + dec_ref(); + if (release) { + PyEval_SaveThread(); + } + } + +private: + PyThreadState *tstate = nullptr; + bool release = true; + bool active = true; +}; + +class gil_scoped_release { +public: + explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + // `get_internals()` must be called here unconditionally in order to initialize + // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an + // initialization race could occur as multiple threads try `gil_scoped_acquire`. + auto &internals = detail::get_internals(); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + tstate = PyEval_SaveThread(); + if (disassoc) { + // Python >= 3.7 can remove this, it's an int before 3.7 + // NOLINTNEXTLINE(readability-qualified-auto) + auto key = internals.tstate; + PYBIND11_TLS_DELETE_VALUE(key); + } + } + + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; + + /// This method will disable the PyThreadState_DeleteCurrent call and the + /// GIL won't be acquired. This method should be used if the interpreter + /// could be shutting down when this is called, as thread deletion is not + /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and + /// protect subsequent code. + PYBIND11_NOINLINE void disarm() { active = false; } + + ~gil_scoped_release() { + if (!tstate) { + return; + } + // `PyEval_RestoreThread()` should not be called if runtime is finalizing + if (active) { + PyEval_RestoreThread(tstate); + } + if (disassoc) { + // Python >= 3.7 can remove this, it's an int before 3.7 + // NOLINTNEXTLINE(readability-qualified-auto) + auto key = detail::get_internals().tstate; + PYBIND11_TLS_REPLACE_VALUE(key, tstate); + } + } + +private: + PyThreadState *tstate; + bool disassoc; + bool active = true; +}; + +# else // PYBIND11_SIMPLE_GIL_MANAGEMENT + +class gil_scoped_acquire { + PyGILState_STATE state; + +public: + gil_scoped_acquire() : state{PyGILState_Ensure()} {} + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; + ~gil_scoped_acquire() { PyGILState_Release(state); } + void disarm() {} +}; + +class gil_scoped_release { + PyThreadState *state; + +public: + gil_scoped_release() : state{PyEval_SaveThread()} {} + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; + ~gil_scoped_release() { PyEval_RestoreThread(state); } + void disarm() {} +}; + +# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT + +#else // WITH_THREAD + +class gil_scoped_acquire { +public: + gil_scoped_acquire() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; + void disarm() {} +}; + +class gil_scoped_release { +public: + gil_scoped_release() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; + void disarm() {} +}; + +#endif // WITH_THREAD + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/iostream.h b/include/pybind11/iostream.h index 9dee7554..1878089e 100644 --- a/include/pybind11/iostream.h +++ b/include/pybind11/iostream.h @@ -5,17 +5,31 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. + + WARNING: The implementation in this file is NOT thread safe. Multiple + threads writing to a redirected ostream concurrently cause data races + and potentially buffer overflows. Therefore it is currently a requirement + that all (possibly) concurrent redirected ostream writes are protected by + a mutex. + #HelpAppreciated: Work on iostream.h thread safety. + For more background see the discussions under + https://github.com/pybind/pybind11/pull/2982 and + https://github.com/pybind/pybind11/pull/2995. */ #pragma once #include "pybind11.h" -#include <streambuf> +#include <algorithm> +#include <cstring> +#include <iostream> +#include <iterator> +#include <memory> #include <ostream> +#include <streambuf> #include <string> -#include <memory> -#include <iostream> +#include <utility> PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -38,54 +52,85 @@ private: return sync() == 0 ? traits_type::not_eof(c) : traits_type::eof(); } - // This function must be non-virtual to be called in a destructor. If the - // rare MSVC test failure shows up with this version, then this should be - // simplified to a fully qualified call. - int _sync() { - if (pbase() != pptr()) { - - { - gil_scoped_acquire tmp; - - // This subtraction cannot be negative, so dropping the sign. - str line(pbase(), static_cast<size_t>(pptr() - pbase())); + // Computes how many bytes at the end of the buffer are part of an + // incomplete sequence of UTF-8 bytes. + // Precondition: pbase() < pptr() + size_t utf8_remainder() const { + const auto rbase = std::reverse_iterator<char *>(pbase()); + const auto rpptr = std::reverse_iterator<char *>(pptr()); + auto is_ascii = [](char c) { return (static_cast<unsigned char>(c) & 0x80) == 0x00; }; + auto is_leading = [](char c) { return (static_cast<unsigned char>(c) & 0xC0) == 0xC0; }; + auto is_leading_2b = [](char c) { return static_cast<unsigned char>(c) <= 0xDF; }; + auto is_leading_3b = [](char c) { return static_cast<unsigned char>(c) <= 0xEF; }; + // If the last character is ASCII, there are no incomplete code points + if (is_ascii(*rpptr)) { + return 0; + } + // Otherwise, work back from the end of the buffer and find the first + // UTF-8 leading byte + const auto rpend = rbase - rpptr >= 3 ? rpptr + 3 : rbase; + const auto leading = std::find_if(rpptr, rpend, is_leading); + if (leading == rbase) { + return 0; + } + const auto dist = static_cast<size_t>(leading - rpptr); + size_t remainder = 0; + + if (dist == 0) { + remainder = 1; // 1-byte code point is impossible + } else if (dist == 1) { + remainder = is_leading_2b(*leading) ? 0 : dist + 1; + } else if (dist == 2) { + remainder = is_leading_3b(*leading) ? 0 : dist + 1; + } + // else if (dist >= 3), at least 4 bytes before encountering an UTF-8 + // leading byte, either no remainder or invalid UTF-8. + // Invalid UTF-8 will cause an exception later when converting + // to a Python string, so that's not handled here. + return remainder; + } - pywrite(line); + // This function must be non-virtual to be called in a destructor. + int _sync() { + if (pbase() != pptr()) { // If buffer is not empty + gil_scoped_acquire tmp; + // This subtraction cannot be negative, so dropping the sign. + auto size = static_cast<size_t>(pptr() - pbase()); + size_t remainder = utf8_remainder(); + + if (size > remainder) { + str line(pbase(), size - remainder); + pywrite(std::move(line)); pyflush(); - - // Placed inside gil_scoped_aquire as a mutex to avoid a race - setp(pbase(), epptr()); } + // Copy the remainder at the end of the buffer to the beginning: + if (remainder > 0) { + std::memmove(pbase(), pptr() - remainder, remainder); + } + setp(pbase(), epptr()); + pbump(static_cast<int>(remainder)); } return 0; } - int sync() override { - return _sync(); - } + int sync() override { return _sync(); } public: - - pythonbuf(object pyostream, size_t buffer_size = 1024) - : buf_size(buffer_size), - d_buffer(new char[buf_size]), - pywrite(pyostream.attr("write")), + explicit pythonbuf(const object &pyostream, size_t buffer_size = 1024) + : buf_size(buffer_size), d_buffer(new char[buf_size]), pywrite(pyostream.attr("write")), pyflush(pyostream.attr("flush")) { setp(d_buffer.get(), d_buffer.get() + buf_size - 1); } - pythonbuf(pythonbuf&&) = default; + pythonbuf(pythonbuf &&) = default; /// Sync before destroy - ~pythonbuf() override { - _sync(); - } + ~pythonbuf() override { _sync(); } }; PYBIND11_NAMESPACE_END(detail) - /** \rst This a move-only guard that redirects output. @@ -106,7 +151,8 @@ PYBIND11_NAMESPACE_END(detail) .. code-block:: cpp { - py::scoped_ostream_redirect output{std::cerr, py::module::import("sys").attr("stderr")}; + py::scoped_ostream_redirect output{ + std::cerr, py::module::import("sys").attr("stderr")}; std::cout << "Hello, World!"; } \endrst */ @@ -117,16 +163,14 @@ protected: detail::pythonbuf buffer; public: - scoped_ostream_redirect( - std::ostream &costream = std::cout, - object pyostream = module_::import("sys").attr("stdout")) + explicit scoped_ostream_redirect(std::ostream &costream = std::cout, + const object &pyostream + = module_::import("sys").attr("stdout")) : costream(costream), buffer(pyostream) { old = costream.rdbuf(&buffer); } - ~scoped_ostream_redirect() { - costream.rdbuf(old); - } + ~scoped_ostream_redirect() { costream.rdbuf(old); } scoped_ostream_redirect(const scoped_ostream_redirect &) = delete; scoped_ostream_redirect(scoped_ostream_redirect &&other) = default; @@ -134,7 +178,6 @@ public: scoped_ostream_redirect &operator=(scoped_ostream_redirect &&) = delete; }; - /** \rst Like `scoped_ostream_redirect`, but redirects cerr by default. This class is provided primary to make ``py::call_guard`` easier to make. @@ -148,13 +191,12 @@ public: \endrst */ class scoped_estream_redirect : public scoped_ostream_redirect { public: - scoped_estream_redirect( - std::ostream &costream = std::cerr, - object pyostream = module_::import("sys").attr("stderr")) - : scoped_ostream_redirect(costream,pyostream) {} + explicit scoped_estream_redirect(std::ostream &costream = std::cerr, + const object &pyostream + = module_::import("sys").attr("stderr")) + : scoped_ostream_redirect(costream, pyostream) {} }; - PYBIND11_NAMESPACE_BEGIN(detail) // Class to redirect output as a context manager. C++ backend. @@ -165,14 +207,16 @@ class OstreamRedirect { std::unique_ptr<scoped_estream_redirect> redirect_stderr; public: - OstreamRedirect(bool do_stdout = true, bool do_stderr = true) + explicit OstreamRedirect(bool do_stdout = true, bool do_stderr = true) : do_stdout_(do_stdout), do_stderr_(do_stderr) {} void enter() { - if (do_stdout_) + if (do_stdout_) { redirect_stdout.reset(new scoped_ostream_redirect()); - if (do_stderr_) + } + if (do_stderr_) { redirect_stderr.reset(new scoped_estream_redirect()); + } } void exit() { @@ -210,11 +254,12 @@ PYBIND11_NAMESPACE_END(detail) m.noisy_function_with_error_printing() \endrst */ -inline class_<detail::OstreamRedirect> add_ostream_redirect(module_ m, std::string name = "ostream_redirect") { - return class_<detail::OstreamRedirect>(m, name.c_str(), module_local()) - .def(init<bool,bool>(), arg("stdout")=true, arg("stderr")=true) +inline class_<detail::OstreamRedirect> +add_ostream_redirect(module_ m, const std::string &name = "ostream_redirect") { + return class_<detail::OstreamRedirect>(std::move(m), name.c_str(), module_local()) + .def(init<bool, bool>(), arg("stdout") = true, arg("stderr") = true) .def("__enter__", &detail::OstreamRedirect::enter) - .def("__exit__", [](detail::OstreamRedirect &self_, args) { self_.exit(); }); + .def("__exit__", [](detail::OstreamRedirect &self_, const args &) { self_.exit(); }); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 019f5688..36077ec0 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -11,24 +11,20 @@ #include "pybind11.h" #include "complex.h" -#include <numeric> + #include <algorithm> #include <array> #include <cstdint> #include <cstdlib> #include <cstring> +#include <functional> +#include <numeric> #include <sstream> #include <string> -#include <functional> #include <type_traits> +#include <typeindex> #include <utility> #include <vector> -#include <typeindex> - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant -#endif /* This will be true on all flat address space platforms and allows us to reduce the whole npy_intp / ssize_t / Py_intptr_t business down to just ssize_t for all size @@ -40,13 +36,19 @@ static_assert(std::is_signed<Py_intptr_t>::value, "Py_intptr_t must be signed"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + class array; // Forward declaration PYBIND11_NAMESPACE_BEGIN(detail) -template <> struct handle_type_name<array> { static constexpr auto name = _("numpy.ndarray"); }; +template <> +struct handle_type_name<array> { + static constexpr auto name = const_name("numpy.ndarray"); +}; -template <typename type, typename SFINAE = void> struct npy_format_descriptor; +template <typename type, typename SFINAE = void> +struct npy_format_descriptor; struct PyArrayDescr_Proxy { PyObject_HEAD @@ -75,51 +77,59 @@ struct PyArray_Proxy { }; struct PyVoidScalarObject_Proxy { - PyObject_VAR_HEAD - char *obval; + PyObject_VAR_HEAD char *obval; PyArrayDescr_Proxy *descr; int flags; PyObject *base; }; struct numpy_type_info { - PyObject* dtype_ptr; + PyObject *dtype_ptr; std::string format_str; }; struct numpy_internals { std::unordered_map<std::type_index, numpy_type_info> registered_dtypes; - numpy_type_info *get_type_info(const std::type_info& tinfo, bool throw_if_missing = true) { + numpy_type_info *get_type_info(const std::type_info &tinfo, bool throw_if_missing = true) { auto it = registered_dtypes.find(std::type_index(tinfo)); - if (it != registered_dtypes.end()) + if (it != registered_dtypes.end()) { return &(it->second); - if (throw_if_missing) + } + if (throw_if_missing) { pybind11_fail(std::string("NumPy type info missing for ") + tinfo.name()); + } return nullptr; } - template<typename T> numpy_type_info *get_type_info(bool throw_if_missing = true) { + template <typename T> + numpy_type_info *get_type_info(bool throw_if_missing = true) { return get_type_info(typeid(typename std::remove_cv<T>::type), throw_if_missing); } }; -inline PYBIND11_NOINLINE void load_numpy_internals(numpy_internals* &ptr) { +PYBIND11_NOINLINE void load_numpy_internals(numpy_internals *&ptr) { ptr = &get_or_create_shared_data<numpy_internals>("_numpy_internals"); } -inline numpy_internals& get_numpy_internals() { - static numpy_internals* ptr = nullptr; - if (!ptr) +inline numpy_internals &get_numpy_internals() { + static numpy_internals *ptr = nullptr; + if (!ptr) { load_numpy_internals(ptr); + } return *ptr; } -template <typename T> struct same_size { - template <typename U> using as = bool_constant<sizeof(T) == sizeof(U)>; +template <typename T> +struct same_size { + template <typename U> + using as = bool_constant<sizeof(T) == sizeof(U)>; }; -template <typename Concrete> constexpr int platform_lookup() { return -1; } +template <typename Concrete> +constexpr int platform_lookup() { + return -1; +} // Lookup a type according to its size, and return a value corresponding to the NumPy typenum. template <typename Concrete, typename T, typename... Ts, typename... Ints> @@ -137,15 +147,26 @@ struct npy_api { NPY_ARRAY_ALIGNED_ = 0x0100, NPY_ARRAY_WRITEABLE_ = 0x0400, NPY_BOOL_ = 0, - NPY_BYTE_, NPY_UBYTE_, - NPY_SHORT_, NPY_USHORT_, - NPY_INT_, NPY_UINT_, - NPY_LONG_, NPY_ULONG_, - NPY_LONGLONG_, NPY_ULONGLONG_, - NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_, - NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_, + NPY_BYTE_, + NPY_UBYTE_, + NPY_SHORT_, + NPY_USHORT_, + NPY_INT_, + NPY_UINT_, + NPY_LONG_, + NPY_ULONG_, + NPY_LONGLONG_, + NPY_ULONGLONG_, + NPY_FLOAT_, + NPY_DOUBLE_, + NPY_LONGDOUBLE_, + NPY_CFLOAT_, + NPY_CDOUBLE_, + NPY_CLONGDOUBLE_, NPY_OBJECT_ = 17, - NPY_STRING_, NPY_UNICODE_, NPY_VOID_, + NPY_STRING_, + NPY_UNICODE_, + NPY_VOID_, // Platform-dependent normalization NPY_INT8_ = NPY_BYTE_, NPY_UINT8_ = NPY_UBYTE_, @@ -154,38 +175,44 @@ struct npy_api { // `npy_common.h` defines the integer aliases. In order, it checks: // NPY_BITSOF_LONG, NPY_BITSOF_LONGLONG, NPY_BITSOF_INT, NPY_BITSOF_SHORT, NPY_BITSOF_CHAR // and assigns the alias to the first matching size, so we should check in this order. - NPY_INT32_ = platform_lookup<std::int32_t, long, int, short>( - NPY_LONG_, NPY_INT_, NPY_SHORT_), + NPY_INT32_ + = platform_lookup<std::int32_t, long, int, short>(NPY_LONG_, NPY_INT_, NPY_SHORT_), NPY_UINT32_ = platform_lookup<std::uint32_t, unsigned long, unsigned int, unsigned short>( NPY_ULONG_, NPY_UINT_, NPY_USHORT_), - NPY_INT64_ = platform_lookup<std::int64_t, long, long long, int>( - NPY_LONG_, NPY_LONGLONG_, NPY_INT_), - NPY_UINT64_ = platform_lookup<std::uint64_t, unsigned long, unsigned long long, unsigned int>( + NPY_INT64_ + = platform_lookup<std::int64_t, long, long long, int>(NPY_LONG_, NPY_LONGLONG_, NPY_INT_), + NPY_UINT64_ + = platform_lookup<std::uint64_t, unsigned long, unsigned long long, unsigned int>( NPY_ULONG_, NPY_ULONGLONG_, NPY_UINT_), }; - typedef struct { + struct PyArray_Dims { Py_intptr_t *ptr; int len; - } PyArray_Dims; + }; - static npy_api& get() { + static npy_api &get() { static npy_api api = lookup(); return api; } bool PyArray_Check_(PyObject *obj) const { - return (bool) PyObject_TypeCheck(obj, PyArray_Type_); + return PyObject_TypeCheck(obj, PyArray_Type_) != 0; } bool PyArrayDescr_Check_(PyObject *obj) const { - return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_); + return PyObject_TypeCheck(obj, PyArrayDescr_Type_) != 0; } unsigned int (*PyArray_GetNDArrayCFeatureVersion_)(); PyObject *(*PyArray_DescrFromType_)(int); - PyObject *(*PyArray_NewFromDescr_) - (PyTypeObject *, PyObject *, int, Py_intptr_t const *, - Py_intptr_t const *, void *, int, PyObject *); + PyObject *(*PyArray_NewFromDescr_)(PyTypeObject *, + PyObject *, + int, + Py_intptr_t const *, + Py_intptr_t const *, + void *, + int, + PyObject *); // Unused. Not removed because that affects ABI of the class. PyObject *(*PyArray_DescrNewFromType_)(int); int (*PyArray_CopyInto_)(PyObject *, PyObject *); @@ -194,15 +221,24 @@ struct npy_api { PyTypeObject *PyVoidArrType_Type_; PyTypeObject *PyArrayDescr_Type_; PyObject *(*PyArray_DescrFromScalar_)(PyObject *); - PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *); - int (*PyArray_DescrConverter_) (PyObject *, PyObject **); - bool (*PyArray_EquivTypes_) (PyObject *, PyObject *); - int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, unsigned char, PyObject **, int *, - Py_intptr_t *, PyObject **, PyObject *); + PyObject *(*PyArray_FromAny_)(PyObject *, PyObject *, int, int, int, PyObject *); + int (*PyArray_DescrConverter_)(PyObject *, PyObject **); + bool (*PyArray_EquivTypes_)(PyObject *, PyObject *); + int (*PyArray_GetArrayParamsFromObject_)(PyObject *, + PyObject *, + unsigned char, + PyObject **, + int *, + Py_intptr_t *, + PyObject **, + PyObject *); PyObject *(*PyArray_Squeeze_)(PyObject *); // Unused. Not removed because that affects ABI of the class. int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); - PyObject* (*PyArray_Resize_)(PyObject*, PyArray_Dims*, int, int); + PyObject *(*PyArray_Resize_)(PyObject *, PyArray_Dims *, int, int); + PyObject *(*PyArray_Newshape_)(PyObject *, PyArray_Dims *, int); + PyObject *(*PyArray_View_)(PyObject *, PyObject *, PyObject *); + private: enum functions { API_PyArray_GetNDArrayCFeatureVersion = 211, @@ -217,26 +253,25 @@ private: API_PyArray_NewCopy = 85, API_PyArray_NewFromDescr = 94, API_PyArray_DescrNewFromType = 96, + API_PyArray_Newshape = 135, + API_PyArray_Squeeze = 136, + API_PyArray_View = 137, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, API_PyArray_GetArrayParamsFromObject = 278, - API_PyArray_Squeeze = 136, API_PyArray_SetBaseObject = 282 }; static npy_api lookup() { module_ m = module_::import("numpy.core.multiarray"); auto c = m.attr("_ARRAY_API"); -#if PY_MAJOR_VERSION >= 3 - void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), NULL); -#else - void **api_ptr = (void **) PyCObject_AsVoidPtr(c.ptr()); -#endif + void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); - if (api.PyArray_GetNDArrayCFeatureVersion_() < 0x7) + if (api.PyArray_GetNDArrayCFeatureVersion_() < 0x7) { pybind11_fail("pybind11 numpy support requires numpy >= 1.7.0"); + } DECL_NPY_API(PyArray_Type); DECL_NPY_API(PyVoidArrType_Type); DECL_NPY_API(PyArrayDescr_Type); @@ -248,96 +283,116 @@ private: DECL_NPY_API(PyArray_NewCopy); DECL_NPY_API(PyArray_NewFromDescr); DECL_NPY_API(PyArray_DescrNewFromType); + DECL_NPY_API(PyArray_Newshape); + DECL_NPY_API(PyArray_Squeeze); + DECL_NPY_API(PyArray_View); DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_GetArrayParamsFromObject); - DECL_NPY_API(PyArray_Squeeze); DECL_NPY_API(PyArray_SetBaseObject); + #undef DECL_NPY_API return api; } }; -inline PyArray_Proxy* array_proxy(void* ptr) { - return reinterpret_cast<PyArray_Proxy*>(ptr); -} +inline PyArray_Proxy *array_proxy(void *ptr) { return reinterpret_cast<PyArray_Proxy *>(ptr); } -inline const PyArray_Proxy* array_proxy(const void* ptr) { - return reinterpret_cast<const PyArray_Proxy*>(ptr); +inline const PyArray_Proxy *array_proxy(const void *ptr) { + return reinterpret_cast<const PyArray_Proxy *>(ptr); } -inline PyArrayDescr_Proxy* array_descriptor_proxy(PyObject* ptr) { - return reinterpret_cast<PyArrayDescr_Proxy*>(ptr); +inline PyArrayDescr_Proxy *array_descriptor_proxy(PyObject *ptr) { + return reinterpret_cast<PyArrayDescr_Proxy *>(ptr); } -inline const PyArrayDescr_Proxy* array_descriptor_proxy(const PyObject* ptr) { - return reinterpret_cast<const PyArrayDescr_Proxy*>(ptr); +inline const PyArrayDescr_Proxy *array_descriptor_proxy(const PyObject *ptr) { + return reinterpret_cast<const PyArrayDescr_Proxy *>(ptr); } -inline bool check_flags(const void* ptr, int flag) { +inline bool check_flags(const void *ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); } -template <typename T> struct is_std_array : std::false_type { }; -template <typename T, size_t N> struct is_std_array<std::array<T, N>> : std::true_type { }; -template <typename T> struct is_complex : std::false_type { }; -template <typename T> struct is_complex<std::complex<T>> : std::true_type { }; +template <typename T> +struct is_std_array : std::false_type {}; +template <typename T, size_t N> +struct is_std_array<std::array<T, N>> : std::true_type {}; +template <typename T> +struct is_complex : std::false_type {}; +template <typename T> +struct is_complex<std::complex<T>> : std::true_type {}; -template <typename T> struct array_info_scalar { +template <typename T> +struct array_info_scalar { using type = T; static constexpr bool is_array = false; static constexpr bool is_empty = false; - static constexpr auto extents = _(""); - static void append_extents(list& /* shape */) { } + static constexpr auto extents = const_name(""); + static void append_extents(list & /* shape */) {} }; // Computes underlying type and a comma-separated list of extents for array // types (any mix of std::array and built-in arrays). An array of char is // treated as scalar because it gets special handling. -template <typename T> struct array_info : array_info_scalar<T> { }; -template <typename T, size_t N> struct array_info<std::array<T, N>> { +template <typename T> +struct array_info : array_info_scalar<T> {}; +template <typename T, size_t N> +struct array_info<std::array<T, N>> { using type = typename array_info<T>::type; static constexpr bool is_array = true; static constexpr bool is_empty = (N == 0) || array_info<T>::is_empty; static constexpr size_t extent = N; // appends the extents to shape - static void append_extents(list& shape) { + static void append_extents(list &shape) { shape.append(N); array_info<T>::append_extents(shape); } - static constexpr auto extents = _<array_info<T>::is_array>( - concat(_<N>(), array_info<T>::extents), _<N>() - ); + static constexpr auto extents = const_name<array_info<T>::is_array>( + concat(const_name<N>(), array_info<T>::extents), const_name<N>()); }; // For numpy we have special handling for arrays of characters, so we don't include // the size in the array extents. -template <size_t N> struct array_info<char[N]> : array_info_scalar<char[N]> { }; -template <size_t N> struct array_info<std::array<char, N>> : array_info_scalar<std::array<char, N>> { }; -template <typename T, size_t N> struct array_info<T[N]> : array_info<std::array<T, N>> { }; -template <typename T> using remove_all_extents_t = typename array_info<T>::type; - -template <typename T> using is_pod_struct = all_of< - std::is_standard_layout<T>, // since we're accessing directly in memory we need a standard layout type -#if !defined(__GNUG__) || defined(_LIBCPP_VERSION) || defined(_GLIBCXX_USE_CXX11_ABI) - // _GLIBCXX_USE_CXX11_ABI indicates that we're using libstdc++ from GCC 5 or newer, independent - // of the actual compiler (Clang can also use libstdc++, but it always defines __GNUC__ == 4). - std::is_trivially_copyable<T>, +template <size_t N> +struct array_info<char[N]> : array_info_scalar<char[N]> {}; +template <size_t N> +struct array_info<std::array<char, N>> : array_info_scalar<std::array<char, N>> {}; +template <typename T, size_t N> +struct array_info<T[N]> : array_info<std::array<T, N>> {}; +template <typename T> +using remove_all_extents_t = typename array_info<T>::type; + +template <typename T> +using is_pod_struct + = all_of<std::is_standard_layout<T>, // since we're accessing directly in memory + // we need a standard layout type +#if defined(__GLIBCXX__) \ + && (__GLIBCXX__ < 20150422 || __GLIBCXX__ == 20150426 || __GLIBCXX__ == 20150623 \ + || __GLIBCXX__ == 20150626 || __GLIBCXX__ == 20160803) + // libstdc++ < 5 (including versions 4.8.5, 4.9.3 and 4.9.4 which were released after + // 5) don't implement is_trivially_copyable, so approximate it + std::is_trivially_destructible<T>, + satisfies_any_of<T, std::has_trivial_copy_constructor, std::has_trivial_copy_assign>, #else - // GCC 4 doesn't implement is_trivially_copyable, so approximate it - std::is_trivially_destructible<T>, - satisfies_any_of<T, std::has_trivial_copy_constructor, std::has_trivial_copy_assign>, + std::is_trivially_copyable<T>, #endif - satisfies_none_of<T, std::is_reference, std::is_array, is_std_array, std::is_arithmetic, is_complex, std::is_enum> ->; + satisfies_none_of<T, + std::is_reference, + std::is_array, + is_std_array, + std::is_arithmetic, + is_complex, + std::is_enum>>; // Replacement for std::is_pod (deprecated in C++20) -template <typename T> using is_pod = all_of< - std::is_standard_layout<T>, - std::is_trivial<T> ->; +template <typename T> +using is_pod = all_of<std::is_standard_layout<T>, std::is_trivial<T>>; -template <ssize_t Dim = 0, typename Strides> ssize_t byte_offset_unsafe(const Strides &) { return 0; } +template <ssize_t Dim = 0, typename Strides> +ssize_t byte_offset_unsafe(const Strides &) { + return 0; +} template <ssize_t Dim = 0, typename Strides, typename... Ix> ssize_t byte_offset_unsafe(const Strides &strides, ssize_t i, Ix... index) { return i * strides[Dim] + byte_offset_unsafe<Dim + 1>(strides, index...); @@ -345,7 +400,7 @@ ssize_t byte_offset_unsafe(const Strides &strides, ssize_t i, Ix... index) { /** * Proxy class providing unsafe, unchecked const access to array data. This is constructed through - * the `unchecked<T, N>()` method of `array` or the `unchecked<N>()` method of `array_t<T>`. `Dims` + * the `unchecked<T, N>()` method of `array` or the `unchecked<N>()` method of `array_t<T>`. `Dims` * will be -1 for dimensions determined at runtime. */ template <typename T, ssize_t Dims> @@ -355,15 +410,17 @@ protected: const unsigned char *data_; // Storing the shape & strides in local variables (i.e. these arrays) allows the compiler to // make large performance gains on big, nested loops, but requires compile-time dimensions - conditional_t<Dynamic, const ssize_t *, std::array<ssize_t, (size_t) Dims>> - shape_, strides_; + conditional_t<Dynamic, const ssize_t *, std::array<ssize_t, (size_t) Dims>> shape_, strides_; const ssize_t dims_; friend class pybind11::array; // Constructor for compile-time dimensions: template <bool Dyn = Dynamic> - unchecked_reference(const void *data, const ssize_t *shape, const ssize_t *strides, enable_if_t<!Dyn, ssize_t>) - : data_{reinterpret_cast<const unsigned char *>(data)}, dims_{Dims} { + unchecked_reference(const void *data, + const ssize_t *shape, + const ssize_t *strides, + enable_if_t<!Dyn, ssize_t>) + : data_{reinterpret_cast<const unsigned char *>(data)}, dims_{Dims} { for (size_t i = 0; i < (size_t) dims_; i++) { shape_[i] = shape[i]; strides_[i] = strides[i]; @@ -371,8 +428,12 @@ protected: } // Constructor for runtime dimensions: template <bool Dyn = Dynamic> - unchecked_reference(const void *data, const ssize_t *shape, const ssize_t *strides, enable_if_t<Dyn, ssize_t> dims) - : data_{reinterpret_cast<const unsigned char *>(data)}, shape_{shape}, strides_{strides}, dims_{dims} {} + unchecked_reference(const void *data, + const ssize_t *shape, + const ssize_t *strides, + enable_if_t<Dyn, ssize_t> dims) + : data_{reinterpret_cast<const unsigned char *>(data)}, shape_{shape}, strides_{strides}, + dims_{dims} {} public: /** @@ -380,20 +441,27 @@ public: * number of dimensions, this requires the correct number of arguments; for run-time * dimensionality, this is not checked (and so is up to the caller to use safely). */ - template <typename... Ix> const T &operator()(Ix... index) const { + template <typename... Ix> + const T &operator()(Ix... index) const { static_assert(ssize_t{sizeof...(Ix)} == Dims || Dynamic, - "Invalid number of indices for unchecked array reference"); - return *reinterpret_cast<const T *>(data_ + byte_offset_unsafe(strides_, ssize_t(index)...)); + "Invalid number of indices for unchecked array reference"); + return *reinterpret_cast<const T *>(data_ + + byte_offset_unsafe(strides_, ssize_t(index)...)); } /** * Unchecked const reference access to data; this operator only participates if the reference * is to a 1-dimensional array. When present, this is exactly equivalent to `obj(index)`. */ template <ssize_t D = Dims, typename = enable_if_t<D == 1 || Dynamic>> - const T &operator[](ssize_t index) const { return operator()(index); } + const T &operator[](ssize_t index) const { + return operator()(index); + } /// Pointer access to the data at the given indices. - template <typename... Ix> const T *data(Ix... ix) const { return &operator()(ssize_t(ix)...); } + template <typename... Ix> + const T *data(Ix... ix) const { + return &operator()(ssize_t(ix)...); + } /// Returns the item size, i.e. sizeof(T) constexpr static ssize_t itemsize() { return sizeof(T); } @@ -404,21 +472,22 @@ public: /// Returns the number of dimensions of the array ssize_t ndim() const { return dims_; } - /// Returns the total number of elements in the referenced array, i.e. the product of the shapes + /// Returns the total number of elements in the referenced array, i.e. the product of the + /// shapes template <bool Dyn = Dynamic> enable_if_t<!Dyn, ssize_t> size() const { - return std::accumulate(shape_.begin(), shape_.end(), (ssize_t) 1, std::multiplies<ssize_t>()); + return std::accumulate( + shape_.begin(), shape_.end(), (ssize_t) 1, std::multiplies<ssize_t>()); } template <bool Dyn = Dynamic> enable_if_t<Dyn, ssize_t> size() const { return std::accumulate(shape_, shape_ + ndim(), (ssize_t) 1, std::multiplies<ssize_t>()); } - /// Returns the total number of bytes used by the referenced data. Note that the actual span in - /// memory may be larger if the referenced array has non-contiguous strides (e.g. for a slice). - ssize_t nbytes() const { - return size() * itemsize(); - } + /// Returns the total number of bytes used by the referenced data. Note that the actual span + /// in memory may be larger if the referenced array has non-contiguous strides (e.g. for a + /// slice). + ssize_t nbytes() const { return size() * itemsize(); } }; template <typename T, ssize_t Dims> @@ -427,15 +496,17 @@ class unchecked_mutable_reference : public unchecked_reference<T, Dims> { using ConstBase = unchecked_reference<T, Dims>; using ConstBase::ConstBase; using ConstBase::Dynamic; + public: // Bring in const-qualified versions from base class using ConstBase::operator(); using ConstBase::operator[]; /// Mutable, unchecked access to data at the given indices. - template <typename... Ix> T& operator()(Ix... index) { + template <typename... Ix> + T &operator()(Ix... index) { static_assert(ssize_t{sizeof...(Ix)} == Dims || Dynamic, - "Invalid number of indices for unchecked array reference"); + "Invalid number of indices for unchecked array reference"); return const_cast<T &>(ConstBase::operator()(index...)); } /** @@ -444,112 +515,169 @@ public: * exactly equivalent to `obj(index)`. */ template <ssize_t D = Dims, typename = enable_if_t<D == 1 || Dynamic>> - T &operator[](ssize_t index) { return operator()(index); } + T &operator[](ssize_t index) { + return operator()(index); + } /// Mutable pointer access to the data at the given indices. - template <typename... Ix> T *mutable_data(Ix... ix) { return &operator()(ssize_t(ix)...); } + template <typename... Ix> + T *mutable_data(Ix... ix) { + return &operator()(ssize_t(ix)...); + } }; template <typename T, ssize_t Dim> struct type_caster<unchecked_reference<T, Dim>> { - static_assert(Dim == 0 && Dim > 0 /* always fail */, "unchecked array proxy object is not castable"); + static_assert(Dim == 0 && Dim > 0 /* always fail */, + "unchecked array proxy object is not castable"); }; template <typename T, ssize_t Dim> -struct type_caster<unchecked_mutable_reference<T, Dim>> : type_caster<unchecked_reference<T, Dim>> {}; +struct type_caster<unchecked_mutable_reference<T, Dim>> + : type_caster<unchecked_reference<T, Dim>> {}; PYBIND11_NAMESPACE_END(detail) class dtype : public object { public: - PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_) explicit dtype(const buffer_info &info) { - dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format))); + dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); // If info.itemsize == 0, use the value calculated from the format string - m_ptr = descr.strip_padding(info.itemsize ? info.itemsize : descr.itemsize()).release().ptr(); + m_ptr = descr.strip_padding(info.itemsize != 0 ? info.itemsize : descr.itemsize()) + .release() + .ptr(); } - explicit dtype(const std::string &format) { - m_ptr = from_args(pybind11::str(format)).release().ptr(); - } + explicit dtype(const pybind11::str &format) : dtype(from_args(format)) {} + + explicit dtype(const std::string &format) : dtype(pybind11::str(format)) {} - dtype(const char *format) : dtype(std::string(format)) { } + explicit dtype(const char *format) : dtype(pybind11::str(format)) {} dtype(list names, list formats, list offsets, ssize_t itemsize) { dict args; - args["names"] = names; - args["formats"] = formats; - args["offsets"] = offsets; + args["names"] = std::move(names); + args["formats"] = std::move(formats); + args["offsets"] = std::move(offsets); args["itemsize"] = pybind11::int_(itemsize); m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType + explicit dtype(int typenum) + : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { + if (m_ptr == nullptr) { + throw error_already_set(); + } + } + /// This is essentially the same as calling numpy.dtype(args) in Python. - static dtype from_args(object args) { + static dtype from_args(const object &args) { PyObject *ptr = nullptr; - if (!detail::npy_api::get().PyArray_DescrConverter_(args.ptr(), &ptr) || !ptr) + if ((detail::npy_api::get().PyArray_DescrConverter_(args.ptr(), &ptr) == 0) || !ptr) { throw error_already_set(); + } return reinterpret_steal<dtype>(ptr); } /// Return dtype associated with a C++ type. - template <typename T> static dtype of() { + template <typename T> + static dtype of() { return detail::npy_format_descriptor<typename std::remove_cv<T>::type>::dtype(); } /// Size of the data type in bytes. - ssize_t itemsize() const { - return detail::array_descriptor_proxy(m_ptr)->elsize; - } + ssize_t itemsize() const { return detail::array_descriptor_proxy(m_ptr)->elsize; } /// Returns true for structured data types. - bool has_fields() const { - return detail::array_descriptor_proxy(m_ptr)->names != nullptr; + bool has_fields() const { return detail::array_descriptor_proxy(m_ptr)->names != nullptr; } + + /// Single-character code for dtype's kind. + /// For example, floating point types are 'f' and integral types are 'i'. + char kind() const { return detail::array_descriptor_proxy(m_ptr)->kind; } + + /// Single-character for dtype's type. + /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'l'. + char char_() const { + // Note: The signature, `dtype::char_` follows the naming of NumPy's + // public Python API (i.e., ``dtype.char``), rather than its internal + // C API (``PyArray_Descr::type``). + return detail::array_descriptor_proxy(m_ptr)->type; } - /// Single-character type code. - char kind() const { - return detail::array_descriptor_proxy(m_ptr)->kind; + /// type number of dtype. + int num() const { + // Note: The signature, `dtype::num` follows the naming of NumPy's public + // Python API (i.e., ``dtype.num``), rather than its internal + // C API (``PyArray_Descr::type_num``). + return detail::array_descriptor_proxy(m_ptr)->type_num; } + /// Single character for byteorder + char byteorder() const { return detail::array_descriptor_proxy(m_ptr)->byteorder; } + + /// Alignment of the data type + int alignment() const { return detail::array_descriptor_proxy(m_ptr)->alignment; } + + /// Flags for the array descriptor + char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; } + private: static object _dtype_from_pep3118() { static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118").cast<object>().release().ptr(); + .attr("_dtype_from_pep3118") + .cast<object>() + .release() + .ptr(); return reinterpret_borrow<object>(obj); } dtype strip_padding(ssize_t itemsize) { // Recursively strip all void fields with empty names that are generated for // padding fields (as of NumPy v1.11). - if (!has_fields()) + if (!has_fields()) { return *this; + } - struct field_descr { PYBIND11_STR_TYPE name; object format; pybind11::int_ offset; }; + struct field_descr { + pybind11::str name; + object format; + pybind11::int_ offset; + field_descr(pybind11::str &&name, object &&format, pybind11::int_ &&offset) + : name{std::move(name)}, format{std::move(format)}, offset{std::move(offset)} {}; + }; + auto field_dict = attr("fields").cast<dict>(); std::vector<field_descr> field_descriptors; + field_descriptors.reserve(field_dict.size()); - for (auto field : attr("fields").attr("items")()) { + for (auto field : field_dict.attr("items")()) { auto spec = field.cast<tuple>(); auto name = spec[0].cast<pybind11::str>(); - auto format = spec[1].cast<tuple>()[0].cast<dtype>(); - auto offset = spec[1].cast<tuple>()[1].cast<pybind11::int_>(); - if (!len(name) && format.kind() == 'V') + auto spec_fo = spec[1].cast<tuple>(); + auto format = spec_fo[0].cast<dtype>(); + auto offset = spec_fo[1].cast<pybind11::int_>(); + if ((len(name) == 0u) && format.kind() == 'V') { continue; - field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(format.itemsize()), offset}); + } + field_descriptors.emplace_back( + std::move(name), format.strip_padding(format.itemsize()), std::move(offset)); } - std::sort(field_descriptors.begin(), field_descriptors.end(), - [](const field_descr& a, const field_descr& b) { + std::sort(field_descriptors.begin(), + field_descriptors.end(), + [](const field_descr &a, const field_descr &b) { return a.offset.cast<int>() < b.offset.cast<int>(); }); list names, formats, offsets; - for (auto& descr : field_descriptors) { - names.append(descr.name); - formats.append(descr.format); - offsets.append(descr.offset); + for (auto &descr : field_descriptors) { + names.append(std::move(descr.name)); + formats.append(std::move(descr.format)); + offsets.append(std::move(descr.offset)); } - return dtype(names, formats, offsets, itemsize); + return dtype(std::move(names), std::move(formats), std::move(offsets), itemsize); } }; @@ -569,66 +697,85 @@ public: using StridesContainer = detail::any_container<ssize_t>; // Constructs an array taking shape/strides from arbitrary container types - array(const pybind11::dtype &dt, ShapeContainer shape, StridesContainer strides, - const void *ptr = nullptr, handle base = handle()) { + array(const pybind11::dtype &dt, + ShapeContainer shape, + StridesContainer strides, + const void *ptr = nullptr, + handle base = handle()) { - if (strides->empty()) + if (strides->empty()) { *strides = detail::c_strides(*shape, dt.itemsize()); + } auto ndim = shape->size(); - if (ndim != strides->size()) + if (ndim != strides->size()) { pybind11_fail("NumPy: shape ndim doesn't match strides ndim"); + } auto descr = dt; int flags = 0; if (base && ptr) { - if (isinstance<array>(base)) + if (isinstance<array>(base)) { /* Copy flags from base (except ownership bit) */ - flags = reinterpret_borrow<array>(base).flags() & ~detail::npy_api::NPY_ARRAY_OWNDATA_; - else + flags = reinterpret_borrow<array>(base).flags() + & ~detail::npy_api::NPY_ARRAY_OWNDATA_; + } else { /* Writable by default, easy to downgrade later on if needed */ flags = detail::npy_api::NPY_ARRAY_WRITEABLE_; + } } auto &api = detail::npy_api::get(); auto tmp = reinterpret_steal<object>(api.PyArray_NewFromDescr_( - api.PyArray_Type_, descr.release().ptr(), (int) ndim, + api.PyArray_Type_, + descr.release().ptr(), + (int) ndim, // Use reinterpret_cast for PyPy on Windows (remove if fixed, checked on 7.3.1) - reinterpret_cast<Py_intptr_t*>(shape->data()), - reinterpret_cast<Py_intptr_t*>(strides->data()), - const_cast<void *>(ptr), flags, nullptr)); - if (!tmp) + reinterpret_cast<Py_intptr_t *>(shape->data()), + reinterpret_cast<Py_intptr_t *>(strides->data()), + const_cast<void *>(ptr), + flags, + nullptr)); + if (!tmp) { throw error_already_set(); + } if (ptr) { if (base) { api.PyArray_SetBaseObject_(tmp.ptr(), base.inc_ref().ptr()); } else { - tmp = reinterpret_steal<object>(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */)); + tmp = reinterpret_steal<object>( + api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */)); } } m_ptr = tmp.release().ptr(); } - array(const pybind11::dtype &dt, ShapeContainer shape, const void *ptr = nullptr, handle base = handle()) - : array(dt, std::move(shape), {}, ptr, base) { } + array(const pybind11::dtype &dt, + ShapeContainer shape, + const void *ptr = nullptr, + handle base = handle()) + : array(dt, std::move(shape), {}, ptr, base) {} - template <typename T, typename = detail::enable_if_t<std::is_integral<T>::value && !std::is_same<bool, T>::value>> + template <typename T, + typename + = detail::enable_if_t<std::is_integral<T>::value && !std::is_same<bool, T>::value>> array(const pybind11::dtype &dt, T count, const void *ptr = nullptr, handle base = handle()) - : array(dt, {{count}}, ptr, base) { } + : array(dt, {{count}}, ptr, base) {} template <typename T> array(ShapeContainer shape, StridesContainer strides, const T *ptr, handle base = handle()) - : array(pybind11::dtype::of<T>(), std::move(shape), std::move(strides), ptr, base) { } + : array(pybind11::dtype::of<T>(), std::move(shape), std::move(strides), ptr, base) {} template <typename T> array(ShapeContainer shape, const T *ptr, handle base = handle()) - : array(std::move(shape), {}, ptr, base) { } + : array(std::move(shape), {}, ptr, base) {} template <typename T> - explicit array(ssize_t count, const T *ptr, handle base = handle()) : array({count}, {}, ptr, base) { } + explicit array(ssize_t count, const T *ptr, handle base = handle()) + : array({count}, {}, ptr, base) {} explicit array(const buffer_info &info, handle base = handle()) - : array(pybind11::dtype(info), info.shape, info.strides, info.ptr, base) { } + : array(pybind11::dtype(info), info.shape, info.strides, info.ptr, base) {} /// Array descriptor (dtype) pybind11::dtype dtype() const { @@ -646,48 +793,38 @@ public: } /// Total number of bytes - ssize_t nbytes() const { - return size() * itemsize(); - } + ssize_t nbytes() const { return size() * itemsize(); } /// Number of dimensions - ssize_t ndim() const { - return detail::array_proxy(m_ptr)->nd; - } + ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; } /// Base object - object base() const { - return reinterpret_borrow<object>(detail::array_proxy(m_ptr)->base); - } + object base() const { return reinterpret_borrow<object>(detail::array_proxy(m_ptr)->base); } /// Dimensions of the array - const ssize_t* shape() const { - return detail::array_proxy(m_ptr)->dimensions; - } + const ssize_t *shape() const { return detail::array_proxy(m_ptr)->dimensions; } /// Dimension along a given axis ssize_t shape(ssize_t dim) const { - if (dim >= ndim()) + if (dim >= ndim()) { fail_dim_check(dim, "invalid axis"); + } return shape()[dim]; } /// Strides of the array - const ssize_t* strides() const { - return detail::array_proxy(m_ptr)->strides; - } + const ssize_t *strides() const { return detail::array_proxy(m_ptr)->strides; } /// Stride along a given axis ssize_t strides(ssize_t dim) const { - if (dim >= ndim()) + if (dim >= ndim()) { fail_dim_check(dim, "invalid axis"); + } return strides()[dim]; } /// Return the NumPy array flags - int flags() const { - return detail::array_proxy(m_ptr)->flags; - } + int flags() const { return detail::array_proxy(m_ptr)->flags; } /// If set, the array is writeable (otherwise the buffer is read-only) bool writeable() const { @@ -701,23 +838,27 @@ public: /// Pointer to the contained data. If index is not provided, points to the /// beginning of the buffer. May throw if the index would lead to out of bounds access. - template<typename... Ix> const void* data(Ix... index) const { + template <typename... Ix> + const void *data(Ix... index) const { return static_cast<const void *>(detail::array_proxy(m_ptr)->data + offset_at(index...)); } /// Mutable pointer to the contained data. If index is not provided, points to the /// beginning of the buffer. May throw if the index would lead to out of bounds access. /// May throw if the array is not writeable. - template<typename... Ix> void* mutable_data(Ix... index) { + template <typename... Ix> + void *mutable_data(Ix... index) { check_writeable(); return static_cast<void *>(detail::array_proxy(m_ptr)->data + offset_at(index...)); } /// Byte offset from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. - template<typename... Ix> ssize_t offset_at(Ix... index) const { - if ((ssize_t) sizeof...(index) > ndim()) + template <typename... Ix> + ssize_t offset_at(Ix... index) const { + if ((ssize_t) sizeof...(index) > ndim()) { fail_dim_check(sizeof...(index), "too many indices for an array"); + } return byte_offset(ssize_t(index)...); } @@ -725,7 +866,8 @@ public: /// Item count from beginning of the array to a given index (full or partial). /// May throw if the index would lead to out of bounds access. - template<typename... Ix> ssize_t index_at(Ix... index) const { + template <typename... Ix> + ssize_t index_at(Ix... index) const { return offset_at(index...) / itemsize(); } @@ -735,30 +877,37 @@ public: * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. */ - template <typename T, ssize_t Dims = -1> detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() & { - if (Dims >= 0 && ndim() != Dims) - throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + - "; expected " + std::to_string(Dims)); - return detail::unchecked_mutable_reference<T, Dims>(mutable_data(), shape(), strides(), ndim()); + template <typename T, ssize_t Dims = -1> + detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() & { + if (Dims >= 0 && ndim() != Dims) { + throw std::domain_error("array has incorrect number of dimensions: " + + std::to_string(ndim()) + "; expected " + + std::to_string(Dims)); + } + return detail::unchecked_mutable_reference<T, Dims>( + mutable_data(), shape(), strides(), ndim()); } /** * Returns a proxy object that provides const access to the array's data without bounds or * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the - * underlying array have the `writable` flag. Use with care: the array must not be destroyed or - * reshaped for the duration of the returned object, and the caller must take care not to access - * invalid dimensions or dimension indices. + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ - template <typename T, ssize_t Dims = -1> detail::unchecked_reference<T, Dims> unchecked() const & { - if (Dims >= 0 && ndim() != Dims) - throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + - "; expected " + std::to_string(Dims)); + template <typename T, ssize_t Dims = -1> + detail::unchecked_reference<T, Dims> unchecked() const & { + if (Dims >= 0 && ndim() != Dims) { + throw std::domain_error("array has incorrect number of dimensions: " + + std::to_string(ndim()) + "; expected " + + std::to_string(Dims)); + } return detail::unchecked_reference<T, Dims>(data(), shape(), strides(), ndim()); } /// Return a new view with all of the dimensions of length 1 removed array squeeze() { - auto& api = detail::npy_api::get(); + auto &api = detail::npy_api::get(); return reinterpret_steal<array>(api.PyArray_Squeeze_(m_ptr)); } @@ -766,57 +915,92 @@ public: /// If refcheck is true and more that one reference exist to this array /// then resize will succeed only if it makes a reshape, i.e. original size doesn't change void resize(ShapeContainer new_shape, bool refcheck = true) { - detail::npy_api::PyArray_Dims d = { - // Use reinterpret_cast for PyPy on Windows (remove if fixed, checked on 7.3.1) - reinterpret_cast<Py_intptr_t*>(new_shape->data()), - int(new_shape->size()) - }; + detail::npy_api::PyArray_Dims d + = {// Use reinterpret_cast for PyPy on Windows (remove if fixed, checked on 7.3.1) + reinterpret_cast<Py_intptr_t *>(new_shape->data()), + int(new_shape->size())}; // try to resize, set ordering param to -1 cause it's not used anyway auto new_array = reinterpret_steal<object>( - detail::npy_api::get().PyArray_Resize_(m_ptr, &d, int(refcheck), -1) - ); - if (!new_array) throw error_already_set(); - if (isinstance<array>(new_array)) { *this = std::move(new_array); } + detail::npy_api::get().PyArray_Resize_(m_ptr, &d, int(refcheck), -1)); + if (!new_array) { + throw error_already_set(); + } + if (isinstance<array>(new_array)) { + *this = std::move(new_array); + } + } + + /// Optional `order` parameter omitted, to be added as needed. + array reshape(ShapeContainer new_shape) { + detail::npy_api::PyArray_Dims d + = {reinterpret_cast<Py_intptr_t *>(new_shape->data()), int(new_shape->size())}; + auto new_array + = reinterpret_steal<array>(detail::npy_api::get().PyArray_Newshape_(m_ptr, &d, 0)); + if (!new_array) { + throw error_already_set(); + } + return new_array; + } + + /// Create a view of an array in a different data type. + /// This function may fundamentally reinterpret the data in the array. + /// It is the responsibility of the caller to ensure that this is safe. + /// Only supports the `dtype` argument, the `type` argument is omitted, + /// to be added as needed. + array view(const std::string &dtype) { + auto &api = detail::npy_api::get(); + auto new_view = reinterpret_steal<array>(api.PyArray_View_( + m_ptr, dtype::from_args(pybind11::str(dtype)).release().ptr(), nullptr)); + if (!new_view) { + throw error_already_set(); + } + return new_view; } /// Ensure that the argument is a NumPy array /// In case of an error, nullptr is returned and the Python error is cleared. static array ensure(handle h, int ExtraFlags = 0) { auto result = reinterpret_steal<array>(raw_array(h.ptr(), ExtraFlags)); - if (!result) + if (!result) { PyErr_Clear(); + } return result; } protected: - template<typename, typename> friend struct detail::npy_format_descriptor; + template <typename, typename> + friend struct detail::npy_format_descriptor; - void fail_dim_check(ssize_t dim, const std::string& msg) const { - throw index_error(msg + ": " + std::to_string(dim) + - " (ndim = " + std::to_string(ndim()) + ")"); + void fail_dim_check(ssize_t dim, const std::string &msg) const { + throw index_error(msg + ": " + std::to_string(dim) + " (ndim = " + std::to_string(ndim()) + + ')'); } - template<typename... Ix> ssize_t byte_offset(Ix... index) const { + template <typename... Ix> + ssize_t byte_offset(Ix... index) const { check_dimensions(index...); return detail::byte_offset_unsafe(strides(), ssize_t(index)...); } void check_writeable() const { - if (!writeable()) + if (!writeable()) { throw std::domain_error("array is not writeable"); + } } - template<typename... Ix> void check_dimensions(Ix... index) const { + template <typename... Ix> + void check_dimensions(Ix... index) const { check_dimensions_impl(ssize_t(0), shape(), ssize_t(index)...); } - void check_dimensions_impl(ssize_t, const ssize_t*) const { } + void check_dimensions_impl(ssize_t, const ssize_t *) const {} - template<typename... Ix> void check_dimensions_impl(ssize_t axis, const ssize_t* shape, ssize_t i, Ix... index) const { + template <typename... Ix> + void check_dimensions_impl(ssize_t axis, const ssize_t *shape, ssize_t i, Ix... index) const { if (i >= *shape) { - throw index_error(std::string("index ") + std::to_string(i) + - " is out of bounds for axis " + std::to_string(axis) + - " with size " + std::to_string(*shape)); + throw index_error(std::string("index ") + std::to_string(i) + + " is out of bounds for axis " + std::to_string(axis) + + " with size " + std::to_string(*shape)); } check_dimensions_impl(axis + 1, shape + 1, index...); } @@ -832,74 +1016,98 @@ protected: } }; -template <typename T, int ExtraFlags = array::forcecast> class array_t : public array { +template <typename T, int ExtraFlags = array::forcecast> +class array_t : public array { private: struct private_ctor {}; // Delegating constructor needed when both moving and accessing in the same constructor - array_t(private_ctor, ShapeContainer &&shape, StridesContainer &&strides, const T *ptr, handle base) + array_t(private_ctor, + ShapeContainer &&shape, + StridesContainer &&strides, + const T *ptr, + handle base) : array(std::move(shape), std::move(strides), ptr, base) {} + public: static_assert(!detail::array_info<T>::is_array, "Array types cannot be used with array_t"); using value_type = T; array_t() : array(0, static_cast<const T *>(nullptr)) {} - array_t(handle h, borrowed_t) : array(h, borrowed_t{}) { } - array_t(handle h, stolen_t) : array(h, stolen_t{}) { } + array_t(handle h, borrowed_t) : array(h, borrowed_t{}) {} + array_t(handle h, stolen_t) : array(h, stolen_t{}) {} PYBIND11_DEPRECATED("Use array_t<T>::ensure() instead") array_t(handle h, bool is_borrowed) : array(raw_array_t(h.ptr()), stolen_t{}) { - if (!m_ptr) PyErr_Clear(); - if (!is_borrowed) Py_XDECREF(h.ptr()); + if (!m_ptr) { + PyErr_Clear(); + } + if (!is_borrowed) { + Py_XDECREF(h.ptr()); + } } + // NOLINTNEXTLINE(google-explicit-constructor) array_t(const object &o) : array(raw_array_t(o.ptr()), stolen_t{}) { - if (!m_ptr) throw error_already_set(); + if (!m_ptr) { + throw error_already_set(); + } } - explicit array_t(const buffer_info& info, handle base = handle()) : array(info, base) { } + explicit array_t(const buffer_info &info, handle base = handle()) : array(info, base) {} - array_t(ShapeContainer shape, StridesContainer strides, const T *ptr = nullptr, handle base = handle()) - : array(std::move(shape), std::move(strides), ptr, base) { } + array_t(ShapeContainer shape, + StridesContainer strides, + const T *ptr = nullptr, + handle base = handle()) + : array(std::move(shape), std::move(strides), ptr, base) {} explicit array_t(ShapeContainer shape, const T *ptr = nullptr, handle base = handle()) - : array_t(private_ctor{}, std::move(shape), - ExtraFlags & f_style - ? detail::f_strides(*shape, itemsize()) - : detail::c_strides(*shape, itemsize()), - ptr, base) { } + : array_t(private_ctor{}, + std::move(shape), + (ExtraFlags & f_style) != 0 ? detail::f_strides(*shape, itemsize()) + : detail::c_strides(*shape, itemsize()), + ptr, + base) {} explicit array_t(ssize_t count, const T *ptr = nullptr, handle base = handle()) - : array({count}, {}, ptr, base) { } + : array({count}, {}, ptr, base) {} - constexpr ssize_t itemsize() const { - return sizeof(T); - } + constexpr ssize_t itemsize() const { return sizeof(T); } - template<typename... Ix> ssize_t index_at(Ix... index) const { + template <typename... Ix> + ssize_t index_at(Ix... index) const { return offset_at(index...) / itemsize(); } - template<typename... Ix> const T* data(Ix... index) const { - return static_cast<const T*>(array::data(index...)); + template <typename... Ix> + const T *data(Ix... index) const { + return static_cast<const T *>(array::data(index...)); } - template<typename... Ix> T* mutable_data(Ix... index) { - return static_cast<T*>(array::mutable_data(index...)); + template <typename... Ix> + T *mutable_data(Ix... index) { + return static_cast<T *>(array::mutable_data(index...)); } // Reference to element at a given index - template<typename... Ix> const T& at(Ix... index) const { - if ((ssize_t) sizeof...(index) != ndim()) + template <typename... Ix> + const T &at(Ix... index) const { + if ((ssize_t) sizeof...(index) != ndim()) { fail_dim_check(sizeof...(index), "index dimension mismatch"); - return *(static_cast<const T*>(array::data()) + byte_offset(ssize_t(index)...) / itemsize()); + } + return *(static_cast<const T *>(array::data()) + + byte_offset(ssize_t(index)...) / itemsize()); } // Mutable reference to element at a given index - template<typename... Ix> T& mutable_at(Ix... index) { - if ((ssize_t) sizeof...(index) != ndim()) + template <typename... Ix> + T &mutable_at(Ix... index) { + if ((ssize_t) sizeof...(index) != ndim()) { fail_dim_check(sizeof...(index), "index dimension mismatch"); - return *(static_cast<T*>(array::mutable_data()) + byte_offset(ssize_t(index)...) / itemsize()); + } + return *(static_cast<T *>(array::mutable_data()) + + byte_offset(ssize_t(index)...) / itemsize()); } /** @@ -908,18 +1116,20 @@ public: * care: the array must not be destroyed or reshaped for the duration of the returned object, * and the caller must take care not to access invalid dimensions or dimension indices. */ - template <ssize_t Dims = -1> detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() & { + template <ssize_t Dims = -1> + detail::unchecked_mutable_reference<T, Dims> mutable_unchecked() & { return array::mutable_unchecked<T, Dims>(); } /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ - template <ssize_t Dims = -1> detail::unchecked_reference<T, Dims> unchecked() const & { + template <ssize_t Dims = -1> + detail::unchecked_reference<T, Dims> unchecked() const & { return array::unchecked<T, Dims>(); } @@ -927,15 +1137,17 @@ public: /// it). In case of an error, nullptr is returned and the Python error is cleared. static array_t ensure(handle h) { auto result = reinterpret_steal<array_t>(raw_array_t(h.ptr())); - if (!result) + if (!result) { PyErr_Clear(); + } return result; } static bool check_(handle h) { const auto &api = detail::npy_api::get(); return api.PyArray_Check_(h.ptr()) - && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, dtype::of<T>().ptr()) + && api.PyArray_EquivTypes_(detail::array_proxy(h.ptr())->descr, + dtype::of<T>().ptr()) && detail::check_flags(h.ptr(), ExtraFlags & (array::c_style | array::f_style)); } @@ -946,9 +1158,13 @@ protected: PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); return nullptr; } - return detail::npy_api::get().PyArray_FromAny_( - ptr, dtype::of<T>().release().ptr(), 0, 0, - detail::npy_api::NPY_ARRAY_ENSUREARRAY_ | ExtraFlags, nullptr); + return detail::npy_api::get().PyArray_FromAny_(ptr, + dtype::of<T>().release().ptr(), + 0, + 0, + detail::npy_api::NPY_ARRAY_ENSUREARRAY_ + | ExtraFlags, + nullptr); } }; @@ -959,11 +1175,13 @@ struct format_descriptor<T, detail::enable_if_t<detail::is_pod_struct<T>::value> } }; -template <size_t N> struct format_descriptor<char[N]> { - static std::string format() { return std::to_string(N) + "s"; } +template <size_t N> +struct format_descriptor<char[N]> { + static std::string format() { return std::to_string(N) + 's'; } }; -template <size_t N> struct format_descriptor<std::array<char, N>> { - static std::string format() { return std::to_string(N) + "s"; } +template <size_t N> +struct format_descriptor<std::array<char, N>> { + static std::string format() { return std::to_string(N) + 's'; } }; template <typename T> @@ -978,7 +1196,7 @@ template <typename T> struct format_descriptor<T, detail::enable_if_t<detail::array_info<T>::is_array>> { static std::string format() { using namespace detail; - static constexpr auto extents = _("(") + array_info<T>::extents + _(")"); + static constexpr auto extents = const_name("(") + array_info<T>::extents + const_name(")"); return extents.text + format_descriptor<remove_all_extents_t<T>>::format(); } }; @@ -989,8 +1207,9 @@ struct pyobject_caster<array_t<T, ExtraFlags>> { using type = array_t<T, ExtraFlags>; bool load(handle src, bool convert) { - if (!convert && !type::check_(src)) + if (!convert && !type::check_(src)) { return false; + } value = type::ensure(src); return static_cast<bool>(value); } @@ -1003,7 +1222,7 @@ struct pyobject_caster<array_t<T, ExtraFlags>> { template <typename T> struct compare_buffer_info<T, detail::enable_if_t<detail::is_pod_struct<T>::value>> { - static bool compare(const buffer_info& b) { + static bool compare(const buffer_info &b) { return npy_api::get().PyArray_EquivTypes_(dtype::of<T>().ptr(), dtype(b).ptr()); } }; @@ -1013,73 +1232,109 @@ struct npy_format_descriptor_name; template <typename T> struct npy_format_descriptor_name<T, enable_if_t<std::is_integral<T>::value>> { - static constexpr auto name = _<std::is_same<T, bool>::value>( - _("bool"), _<std::is_signed<T>::value>("numpy.int", "numpy.uint") + _<sizeof(T)*8>() - ); + static constexpr auto name = const_name<std::is_same<T, bool>::value>( + const_name("bool"), + const_name<std::is_signed<T>::value>("numpy.int", "numpy.uint") + + const_name<sizeof(T) * 8>()); }; template <typename T> struct npy_format_descriptor_name<T, enable_if_t<std::is_floating_point<T>::value>> { - static constexpr auto name = _<std::is_same<T, float>::value || std::is_same<T, double>::value>( - _("numpy.float") + _<sizeof(T)*8>(), _("numpy.longdouble") - ); + static constexpr auto name = const_name < std::is_same<T, float>::value + || std::is_same<T, const float>::value + || std::is_same<T, double>::value + || std::is_same<T, const double>::value + > (const_name("numpy.float") + const_name<sizeof(T) * 8>(), + const_name("numpy.longdouble")); }; template <typename T> struct npy_format_descriptor_name<T, enable_if_t<is_complex<T>::value>> { - static constexpr auto name = _<std::is_same<typename T::value_type, float>::value - || std::is_same<typename T::value_type, double>::value>( - _("numpy.complex") + _<sizeof(typename T::value_type)*16>(), _("numpy.longcomplex") - ); + static constexpr auto name = const_name < std::is_same<typename T::value_type, float>::value + || std::is_same<typename T::value_type, const float>::value + || std::is_same<typename T::value_type, double>::value + || std::is_same<typename T::value_type, const double>::value + > (const_name("numpy.complex") + + const_name<sizeof(typename T::value_type) * 16>(), + const_name("numpy.longcomplex")); }; template <typename T> -struct npy_format_descriptor<T, enable_if_t<satisfies_any_of<T, std::is_arithmetic, is_complex>::value>> +struct npy_format_descriptor< + T, + enable_if_t<satisfies_any_of<T, std::is_arithmetic, is_complex>::value>> : npy_format_descriptor_name<T> { private: // NB: the order here must match the one in common.h - constexpr static const int values[15] = { - npy_api::NPY_BOOL_, - npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_INT16_, npy_api::NPY_UINT16_, - npy_api::NPY_INT32_, npy_api::NPY_UINT32_, npy_api::NPY_INT64_, npy_api::NPY_UINT64_, - npy_api::NPY_FLOAT_, npy_api::NPY_DOUBLE_, npy_api::NPY_LONGDOUBLE_, - npy_api::NPY_CFLOAT_, npy_api::NPY_CDOUBLE_, npy_api::NPY_CLONGDOUBLE_ - }; + constexpr static const int values[15] = {npy_api::NPY_BOOL_, + npy_api::NPY_BYTE_, + npy_api::NPY_UBYTE_, + npy_api::NPY_INT16_, + npy_api::NPY_UINT16_, + npy_api::NPY_INT32_, + npy_api::NPY_UINT32_, + npy_api::NPY_INT64_, + npy_api::NPY_UINT64_, + npy_api::NPY_FLOAT_, + npy_api::NPY_DOUBLE_, + npy_api::NPY_LONGDOUBLE_, + npy_api::NPY_CFLOAT_, + npy_api::NPY_CDOUBLE_, + npy_api::NPY_CLONGDOUBLE_}; public: static constexpr int value = values[detail::is_fmt_numeric<T>::index]; - static pybind11::dtype dtype() { - if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) - return reinterpret_steal<pybind11::dtype>(ptr); - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; -#define PYBIND11_DECL_CHAR_FMT \ - static constexpr auto name = _("S") + _<N>(); \ - static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); } -template <size_t N> struct npy_format_descriptor<char[N]> { PYBIND11_DECL_CHAR_FMT }; -template <size_t N> struct npy_format_descriptor<std::array<char, N>> { PYBIND11_DECL_CHAR_FMT }; +template <typename T> +struct npy_format_descriptor<T, enable_if_t<is_same_ignoring_cvref<T, PyObject *>::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +#define PYBIND11_DECL_CHAR_FMT \ + static constexpr auto name = const_name("S") + const_name<N>(); \ + static pybind11::dtype dtype() { \ + return pybind11::dtype(std::string("S") + std::to_string(N)); \ + } +template <size_t N> +struct npy_format_descriptor<char[N]> { + PYBIND11_DECL_CHAR_FMT +}; +template <size_t N> +struct npy_format_descriptor<std::array<char, N>> { + PYBIND11_DECL_CHAR_FMT +}; #undef PYBIND11_DECL_CHAR_FMT -template<typename T> struct npy_format_descriptor<T, enable_if_t<array_info<T>::is_array>> { +template <typename T> +struct npy_format_descriptor<T, enable_if_t<array_info<T>::is_array>> { private: using base_descr = npy_format_descriptor<typename array_info<T>::type>; + public: static_assert(!array_info<T>::is_empty, "Zero-sized arrays are not supported"); - static constexpr auto name = _("(") + array_info<T>::extents + _(")") + base_descr::name; + static constexpr auto name + = const_name("(") + array_info<T>::extents + const_name(")") + base_descr::name; static pybind11::dtype dtype() { list shape; array_info<T>::append_extents(shape); - return pybind11::dtype::from_args(pybind11::make_tuple(base_descr::dtype(), shape)); + return pybind11::dtype::from_args( + pybind11::make_tuple(base_descr::dtype(), std::move(shape))); } }; -template<typename T> struct npy_format_descriptor<T, enable_if_t<std::is_enum<T>::value>> { +template <typename T> +struct npy_format_descriptor<T, enable_if_t<std::is_enum<T>::value>> { private: using base_descr = npy_format_descriptor<typename std::underlying_type<T>::type>; + public: static constexpr auto name = base_descr::name; static pybind11::dtype dtype() { return base_descr::dtype(); } @@ -1093,31 +1348,38 @@ struct field_descriptor { dtype descr; }; -inline PYBIND11_NOINLINE void register_structured_dtype( - any_container<field_descriptor> fields, - const std::type_info& tinfo, ssize_t itemsize, - bool (*direct_converter)(PyObject *, void *&)) { +PYBIND11_NOINLINE void register_structured_dtype(any_container<field_descriptor> fields, + const std::type_info &tinfo, + ssize_t itemsize, + bool (*direct_converter)(PyObject *, void *&)) { - auto& numpy_internals = get_numpy_internals(); - if (numpy_internals.get_type_info(tinfo, false)) + auto &numpy_internals = get_numpy_internals(); + if (numpy_internals.get_type_info(tinfo, false)) { pybind11_fail("NumPy: dtype is already registered"); + } // Use ordered fields because order matters as of NumPy 1.14: // https://docs.scipy.org/doc/numpy/release.html#multiple-field-indexing-assignment-of-structured-arrays std::vector<field_descriptor> ordered_fields(std::move(fields)); - std::sort(ordered_fields.begin(), ordered_fields.end(), + std::sort( + ordered_fields.begin(), + ordered_fields.end(), [](const field_descriptor &a, const field_descriptor &b) { return a.offset < b.offset; }); list names, formats, offsets; - for (auto& field : ordered_fields) { - if (!field.descr) - pybind11_fail(std::string("NumPy: unsupported field dtype: `") + - field.name + "` @ " + tinfo.name()); - names.append(PYBIND11_STR_TYPE(field.name)); + for (auto &field : ordered_fields) { + if (!field.descr) { + pybind11_fail(std::string("NumPy: unsupported field dtype: `") + field.name + "` @ " + + tinfo.name()); + } + names.append(pybind11::str(field.name)); formats.append(field.descr); offsets.append(pybind11::int_(field.offset)); } - auto dtype_ptr = pybind11::dtype(names, formats, offsets, itemsize).release().ptr(); + auto *dtype_ptr + = pybind11::dtype(std::move(names), std::move(formats), std::move(offsets), itemsize) + .release() + .ptr(); // There is an existing bug in NumPy (as of v1.11): trailing bytes are // not encoded explicitly into the format string. This will supposedly @@ -1134,36 +1396,39 @@ inline PYBIND11_NOINLINE void register_structured_dtype( // overriding the endianness. Putting the ^ in front of individual fields // isn't guaranteed to work due to https://github.com/numpy/numpy/issues/9049 oss << "^T{"; - for (auto& field : ordered_fields) { - if (field.offset > offset) + for (auto &field : ordered_fields) { + if (field.offset > offset) { oss << (field.offset - offset) << 'x'; + } oss << field.format << ':' << field.name << ':'; offset = field.offset + field.size; } - if (itemsize > offset) + if (itemsize > offset) { oss << (itemsize - offset) << 'x'; + } oss << '}'; auto format_str = oss.str(); - // Sanity check: verify that NumPy properly parses our buffer format string - auto& api = npy_api::get(); - auto arr = array(buffer_info(nullptr, itemsize, format_str, 1)); - if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) + // Smoke test: verify that NumPy properly parses our buffer format string + auto &api = npy_api::get(); + auto arr = array(buffer_info(nullptr, itemsize, format_str, 1)); + if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) { pybind11_fail("NumPy: invalid buffer descriptor!"); + } auto tindex = std::type_index(tinfo); - numpy_internals.registered_dtypes[tindex] = { dtype_ptr, format_str }; + numpy_internals.registered_dtypes[tindex] = {dtype_ptr, std::move(format_str)}; get_internals().direct_conversions[tindex].push_back(direct_converter); } -template <typename T, typename SFINAE> struct npy_format_descriptor { - static_assert(is_pod_struct<T>::value, "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); +template <typename T, typename SFINAE> +struct npy_format_descriptor { + static_assert(is_pod_struct<T>::value, + "Attempt to use a non-POD or unimplemented POD type as a numpy dtype"); static constexpr auto name = make_caster<T>::name; - static pybind11::dtype dtype() { - return reinterpret_borrow<pybind11::dtype>(dtype_ptr()); - } + static pybind11::dtype dtype() { return reinterpret_borrow<pybind11::dtype>(dtype_ptr()); } static std::string format() { static auto format_str = get_numpy_internals().get_type_info<T>(true)->format_str; @@ -1171,20 +1436,23 @@ template <typename T, typename SFINAE> struct npy_format_descriptor { } static void register_dtype(any_container<field_descriptor> fields) { - register_structured_dtype(std::move(fields), typeid(typename std::remove_cv<T>::type), - sizeof(T), &direct_converter); + register_structured_dtype(std::move(fields), + typeid(typename std::remove_cv<T>::type), + sizeof(T), + &direct_converter); } private: - static PyObject* dtype_ptr() { - static PyObject* ptr = get_numpy_internals().get_type_info<T>(true)->dtype_ptr; + static PyObject *dtype_ptr() { + static PyObject *ptr = get_numpy_internals().get_type_info<T>(true)->dtype_ptr; return ptr; } - static bool direct_converter(PyObject *obj, void*& value) { - auto& api = npy_api::get(); - if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) + static bool direct_converter(PyObject *obj, void *&value) { + auto &api = npy_api::get(); + if (!PyObject_TypeCheck(obj, api.PyVoidArrType_Type_)) { return false; + } if (auto descr = reinterpret_steal<object>(api.PyArray_DescrFromScalar_(obj))) { if (api.PyArray_EquivTypes_(dtype_ptr(), descr.ptr())) { value = ((PyVoidScalarObject_Proxy *) obj)->obval; @@ -1196,78 +1464,80 @@ private: }; #ifdef __CLION_IDE__ // replace heavy macro with dummy code for the IDE (doesn't affect code) -# define PYBIND11_NUMPY_DTYPE(Type, ...) ((void)0) -# define PYBIND11_NUMPY_DTYPE_EX(Type, ...) ((void)0) +# define PYBIND11_NUMPY_DTYPE(Type, ...) ((void) 0) +# define PYBIND11_NUMPY_DTYPE_EX(Type, ...) ((void) 0) #else -#define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ - ::pybind11::detail::field_descriptor { \ - Name, offsetof(T, Field), sizeof(decltype(std::declval<T>().Field)), \ - ::pybind11::format_descriptor<decltype(std::declval<T>().Field)>::format(), \ - ::pybind11::detail::npy_format_descriptor<decltype(std::declval<T>().Field)>::dtype() \ - } +# define PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, Name) \ + ::pybind11::detail::field_descriptor { \ + Name, offsetof(T, Field), sizeof(decltype(std::declval<T>().Field)), \ + ::pybind11::format_descriptor<decltype(std::declval<T>().Field)>::format(), \ + ::pybind11::detail::npy_format_descriptor< \ + decltype(std::declval<T>().Field)>::dtype() \ + } // Extract name, offset and format descriptor for a struct field -#define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) +# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz -#define PYBIND11_EVAL0(...) __VA_ARGS__ -#define PYBIND11_EVAL1(...) PYBIND11_EVAL0 (PYBIND11_EVAL0 (PYBIND11_EVAL0 (__VA_ARGS__))) -#define PYBIND11_EVAL2(...) PYBIND11_EVAL1 (PYBIND11_EVAL1 (PYBIND11_EVAL1 (__VA_ARGS__))) -#define PYBIND11_EVAL3(...) PYBIND11_EVAL2 (PYBIND11_EVAL2 (PYBIND11_EVAL2 (__VA_ARGS__))) -#define PYBIND11_EVAL4(...) PYBIND11_EVAL3 (PYBIND11_EVAL3 (PYBIND11_EVAL3 (__VA_ARGS__))) -#define PYBIND11_EVAL(...) PYBIND11_EVAL4 (PYBIND11_EVAL4 (PYBIND11_EVAL4 (__VA_ARGS__))) -#define PYBIND11_MAP_END(...) -#define PYBIND11_MAP_OUT -#define PYBIND11_MAP_COMMA , -#define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END -#define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT -#define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0) -#define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next) -#if defined(_MSC_VER) && !defined(__clang__) // MSVC is not as eager to expand macros, hence this workaround -#define PYBIND11_MAP_LIST_NEXT1(test, next) \ - PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) -#else -#define PYBIND11_MAP_LIST_NEXT1(test, next) \ - PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0) -#endif -#define PYBIND11_MAP_LIST_NEXT(test, next) \ - PYBIND11_MAP_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next) -#define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \ - f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__) -#define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ - f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__) +# define PYBIND11_EVAL0(...) __VA_ARGS__ +# define PYBIND11_EVAL1(...) PYBIND11_EVAL0(PYBIND11_EVAL0(PYBIND11_EVAL0(__VA_ARGS__))) +# define PYBIND11_EVAL2(...) PYBIND11_EVAL1(PYBIND11_EVAL1(PYBIND11_EVAL1(__VA_ARGS__))) +# define PYBIND11_EVAL3(...) PYBIND11_EVAL2(PYBIND11_EVAL2(PYBIND11_EVAL2(__VA_ARGS__))) +# define PYBIND11_EVAL4(...) PYBIND11_EVAL3(PYBIND11_EVAL3(PYBIND11_EVAL3(__VA_ARGS__))) +# define PYBIND11_EVAL(...) PYBIND11_EVAL4(PYBIND11_EVAL4(PYBIND11_EVAL4(__VA_ARGS__))) +# define PYBIND11_MAP_END(...) +# define PYBIND11_MAP_OUT +# define PYBIND11_MAP_COMMA , +# define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END +# define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT +# define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0(test, next, 0) +# define PYBIND11_MAP_NEXT(test, next) PYBIND11_MAP_NEXT1(PYBIND11_MAP_GET_END test, next) +# if defined(_MSC_VER) \ + && !defined(__clang__) // MSVC is not as eager to expand macros, hence this workaround +# define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_EVAL0(PYBIND11_MAP_NEXT0(test, PYBIND11_MAP_COMMA next, 0)) +# else +# define PYBIND11_MAP_LIST_NEXT1(test, next) \ + PYBIND11_MAP_NEXT0(test, PYBIND11_MAP_COMMA next, 0) +# endif +# define PYBIND11_MAP_LIST_NEXT(test, next) \ + PYBIND11_MAP_LIST_NEXT1(PYBIND11_MAP_GET_END test, next) +# define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT(peek, PYBIND11_MAP_LIST1)(f, t, peek, __VA_ARGS__) +# define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \ + f(t, x) PYBIND11_MAP_LIST_NEXT(peek, PYBIND11_MAP_LIST0)(f, t, peek, __VA_ARGS__) // PYBIND11_MAP_LIST(f, t, a1, a2, ...) expands to f(t, a1), f(t, a2), ... -#define PYBIND11_MAP_LIST(f, t, ...) \ - PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0)) - -#define PYBIND11_NUMPY_DTYPE(Type, ...) \ - ::pybind11::detail::npy_format_descriptor<Type>::register_dtype \ - (::std::vector<::pybind11::detail::field_descriptor> \ - {PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) - -#if defined(_MSC_VER) && !defined(__clang__) -#define PYBIND11_MAP2_LIST_NEXT1(test, next) \ - PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)) -#else -#define PYBIND11_MAP2_LIST_NEXT1(test, next) \ - PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0) -#endif -#define PYBIND11_MAP2_LIST_NEXT(test, next) \ - PYBIND11_MAP2_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next) -#define PYBIND11_MAP2_LIST0(f, t, x1, x2, peek, ...) \ - f(t, x1, x2) PYBIND11_MAP2_LIST_NEXT (peek, PYBIND11_MAP2_LIST1) (f, t, peek, __VA_ARGS__) -#define PYBIND11_MAP2_LIST1(f, t, x1, x2, peek, ...) \ - f(t, x1, x2) PYBIND11_MAP2_LIST_NEXT (peek, PYBIND11_MAP2_LIST0) (f, t, peek, __VA_ARGS__) +# define PYBIND11_MAP_LIST(f, t, ...) \ + PYBIND11_EVAL(PYBIND11_MAP_LIST1(f, t, __VA_ARGS__, (), 0)) + +# define PYBIND11_NUMPY_DTYPE(Type, ...) \ + ::pybind11::detail::npy_format_descriptor<Type>::register_dtype( \ + ::std::vector<::pybind11::detail::field_descriptor>{ \ + PYBIND11_MAP_LIST(PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)}) + +# if defined(_MSC_VER) && !defined(__clang__) +# define PYBIND11_MAP2_LIST_NEXT1(test, next) \ + PYBIND11_EVAL0(PYBIND11_MAP_NEXT0(test, PYBIND11_MAP_COMMA next, 0)) +# else +# define PYBIND11_MAP2_LIST_NEXT1(test, next) \ + PYBIND11_MAP_NEXT0(test, PYBIND11_MAP_COMMA next, 0) +# endif +# define PYBIND11_MAP2_LIST_NEXT(test, next) \ + PYBIND11_MAP2_LIST_NEXT1(PYBIND11_MAP_GET_END test, next) +# define PYBIND11_MAP2_LIST0(f, t, x1, x2, peek, ...) \ + f(t, x1, x2) PYBIND11_MAP2_LIST_NEXT(peek, PYBIND11_MAP2_LIST1)(f, t, peek, __VA_ARGS__) +# define PYBIND11_MAP2_LIST1(f, t, x1, x2, peek, ...) \ + f(t, x1, x2) PYBIND11_MAP2_LIST_NEXT(peek, PYBIND11_MAP2_LIST0)(f, t, peek, __VA_ARGS__) // PYBIND11_MAP2_LIST(f, t, a1, a2, ...) expands to f(t, a1, a2), f(t, a3, a4), ... -#define PYBIND11_MAP2_LIST(f, t, ...) \ - PYBIND11_EVAL (PYBIND11_MAP2_LIST1 (f, t, __VA_ARGS__, (), 0)) +# define PYBIND11_MAP2_LIST(f, t, ...) \ + PYBIND11_EVAL(PYBIND11_MAP2_LIST1(f, t, __VA_ARGS__, (), 0)) -#define PYBIND11_NUMPY_DTYPE_EX(Type, ...) \ - ::pybind11::detail::npy_format_descriptor<Type>::register_dtype \ - (::std::vector<::pybind11::detail::field_descriptor> \ - {PYBIND11_MAP2_LIST (PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) +# define PYBIND11_NUMPY_DTYPE_EX(Type, ...) \ + ::pybind11::detail::npy_format_descriptor<Type>::register_dtype( \ + ::std::vector<::pybind11::detail::field_descriptor>{ \ + PYBIND11_MAP2_LIST(PYBIND11_FIELD_DESCRIPTOR_EX, Type, __VA_ARGS__)}) #endif // __CLION_IDE__ @@ -1277,10 +1547,10 @@ public: using value_type = container_type::value_type; using size_type = container_type::size_type; - common_iterator() : p_ptr(0), m_strides() {} + common_iterator() : m_strides() {} - common_iterator(void* ptr, const container_type& strides, const container_type& shape) - : p_ptr(reinterpret_cast<char*>(ptr)), m_strides(strides.size()) { + common_iterator(void *ptr, const container_type &strides, const container_type &shape) + : p_ptr(reinterpret_cast<char *>(ptr)), m_strides(strides.size()) { m_strides.back() = static_cast<value_type>(strides.back()); for (size_type i = m_strides.size() - 1; i != 0; --i) { size_type j = i - 1; @@ -1289,56 +1559,52 @@ public: } } - void increment(size_type dim) { - p_ptr += m_strides[dim]; - } + void increment(size_type dim) { p_ptr += m_strides[dim]; } - void* data() const { - return p_ptr; - } + void *data() const { return p_ptr; } private: - char* p_ptr; + char *p_ptr{nullptr}; container_type m_strides; }; -template <size_t N> class multi_array_iterator { +template <size_t N> +class multi_array_iterator { public: using container_type = std::vector<ssize_t>; - multi_array_iterator(const std::array<buffer_info, N> &buffers, - const container_type &shape) - : m_shape(shape.size()), m_index(shape.size(), 0), - m_common_iterator() { + multi_array_iterator(const std::array<buffer_info, N> &buffers, const container_type &shape) + : m_shape(shape.size()), m_index(shape.size(), 0), m_common_iterator() { // Manual copy to avoid conversion warning if using std::copy - for (size_t i = 0; i < shape.size(); ++i) + for (size_t i = 0; i < shape.size(); ++i) { m_shape[i] = shape[i]; + } container_type strides(shape.size()); - for (size_t i = 0; i < N; ++i) + for (size_t i = 0; i < N; ++i) { init_common_iterator(buffers[i], shape, m_common_iterator[i], strides); + } } - multi_array_iterator& operator++() { + multi_array_iterator &operator++() { for (size_t j = m_index.size(); j != 0; --j) { size_t i = j - 1; if (++m_index[i] != m_shape[i]) { increment_common_iterator(i); break; - } else { - m_index[i] = 0; } + m_index[i] = 0; } return *this; } - template <size_t K, class T = void> T* data() const { - return reinterpret_cast<T*>(m_common_iterator[K].data()); + template <size_t K, class T = void> + T *data() const { + return reinterpret_cast<T *>(m_common_iterator[K].data()); } private: - using common_iter = common_iterator; void init_common_iterator(const buffer_info &buffer, @@ -1351,10 +1617,11 @@ private: auto strides_iter = strides.rbegin(); while (buffer_shape_iter != buffer.shape.rend()) { - if (*shape_iter == *buffer_shape_iter) + if (*shape_iter == *buffer_shape_iter) { *strides_iter = *buffer_strides_iter; - else + } else { *strides_iter = 0; + } ++buffer_shape_iter; ++buffer_strides_iter; @@ -1367,8 +1634,9 @@ private: } void increment_common_iterator(size_t dim) { - for (auto &iter : m_common_iterator) + for (auto &iter : m_common_iterator) { iter.increment(dim); + } } container_type m_shape; @@ -1378,60 +1646,71 @@ private: enum class broadcast_trivial { non_trivial, c_trivial, f_trivial }; -// Populates the shape and number of dimensions for the set of buffers. Returns a broadcast_trivial -// enum value indicating whether the broadcast is "trivial"--that is, has each buffer being either a -// singleton or a full-size, C-contiguous (`c_trivial`) or Fortran-contiguous (`f_trivial`) storage -// buffer; returns `non_trivial` otherwise. +// Populates the shape and number of dimensions for the set of buffers. Returns a +// broadcast_trivial enum value indicating whether the broadcast is "trivial"--that is, has each +// buffer being either a singleton or a full-size, C-contiguous (`c_trivial`) or Fortran-contiguous +// (`f_trivial`) storage buffer; returns `non_trivial` otherwise. template <size_t N> -broadcast_trivial broadcast(const std::array<buffer_info, N> &buffers, ssize_t &ndim, std::vector<ssize_t> &shape) { - ndim = std::accumulate(buffers.begin(), buffers.end(), ssize_t(0), [](ssize_t res, const buffer_info &buf) { - return std::max(res, buf.ndim); - }); +broadcast_trivial +broadcast(const std::array<buffer_info, N> &buffers, ssize_t &ndim, std::vector<ssize_t> &shape) { + ndim = std::accumulate( + buffers.begin(), buffers.end(), ssize_t(0), [](ssize_t res, const buffer_info &buf) { + return std::max(res, buf.ndim); + }); shape.clear(); shape.resize((size_t) ndim, 1); - // Figure out the output size, and make sure all input arrays conform (i.e. are either size 1 or - // the full size). + // Figure out the output size, and make sure all input arrays conform (i.e. are either size 1 + // or the full size). for (size_t i = 0; i < N; ++i) { auto res_iter = shape.rbegin(); auto end = buffers[i].shape.rend(); - for (auto shape_iter = buffers[i].shape.rbegin(); shape_iter != end; ++shape_iter, ++res_iter) { + for (auto shape_iter = buffers[i].shape.rbegin(); shape_iter != end; + ++shape_iter, ++res_iter) { const auto &dim_size_in = *shape_iter; auto &dim_size_out = *res_iter; - // Each input dimension can either be 1 or `n`, but `n` values must match across buffers - if (dim_size_out == 1) + // Each input dimension can either be 1 or `n`, but `n` values must match across + // buffers + if (dim_size_out == 1) { dim_size_out = dim_size_in; - else if (dim_size_in != 1 && dim_size_in != dim_size_out) + } else if (dim_size_in != 1 && dim_size_in != dim_size_out) { pybind11_fail("pybind11::vectorize: incompatible size/dimension of inputs!"); + } } } bool trivial_broadcast_c = true; bool trivial_broadcast_f = true; for (size_t i = 0; i < N && (trivial_broadcast_c || trivial_broadcast_f); ++i) { - if (buffers[i].size == 1) + if (buffers[i].size == 1) { continue; + } // Require the same number of dimensions: - if (buffers[i].ndim != ndim) + if (buffers[i].ndim != ndim) { return broadcast_trivial::non_trivial; + } // Require all dimensions be full-size: - if (!std::equal(buffers[i].shape.cbegin(), buffers[i].shape.cend(), shape.cbegin())) + if (!std::equal(buffers[i].shape.cbegin(), buffers[i].shape.cend(), shape.cbegin())) { return broadcast_trivial::non_trivial; + } // Check for C contiguity (but only if previous inputs were also C contiguous) if (trivial_broadcast_c) { ssize_t expect_stride = buffers[i].itemsize; auto end = buffers[i].shape.crend(); - for (auto shape_iter = buffers[i].shape.crbegin(), stride_iter = buffers[i].strides.crbegin(); - trivial_broadcast_c && shape_iter != end; ++shape_iter, ++stride_iter) { - if (expect_stride == *stride_iter) + for (auto shape_iter = buffers[i].shape.crbegin(), + stride_iter = buffers[i].strides.crbegin(); + trivial_broadcast_c && shape_iter != end; + ++shape_iter, ++stride_iter) { + if (expect_stride == *stride_iter) { expect_stride *= *shape_iter; - else + } else { trivial_broadcast_c = false; + } } } @@ -1439,61 +1718,61 @@ broadcast_trivial broadcast(const std::array<buffer_info, N> &buffers, ssize_t & if (trivial_broadcast_f) { ssize_t expect_stride = buffers[i].itemsize; auto end = buffers[i].shape.cend(); - for (auto shape_iter = buffers[i].shape.cbegin(), stride_iter = buffers[i].strides.cbegin(); - trivial_broadcast_f && shape_iter != end; ++shape_iter, ++stride_iter) { - if (expect_stride == *stride_iter) + for (auto shape_iter = buffers[i].shape.cbegin(), + stride_iter = buffers[i].strides.cbegin(); + trivial_broadcast_f && shape_iter != end; + ++shape_iter, ++stride_iter) { + if (expect_stride == *stride_iter) { expect_stride *= *shape_iter; - else + } else { trivial_broadcast_f = false; + } } } } - return - trivial_broadcast_c ? broadcast_trivial::c_trivial : - trivial_broadcast_f ? broadcast_trivial::f_trivial : - broadcast_trivial::non_trivial; + return trivial_broadcast_c ? broadcast_trivial::c_trivial + : trivial_broadcast_f ? broadcast_trivial::f_trivial + : broadcast_trivial::non_trivial; } template <typename T> struct vectorize_arg { - static_assert(!std::is_rvalue_reference<T>::value, "Functions with rvalue reference arguments cannot be vectorized"); + static_assert(!std::is_rvalue_reference<T>::value, + "Functions with rvalue reference arguments cannot be vectorized"); // The wrapped function gets called with this type: using call_type = remove_reference_t<T>; // Is this a vectorized argument? - static constexpr bool vectorize = - satisfies_any_of<call_type, std::is_arithmetic, is_complex, is_pod>::value && - satisfies_none_of<call_type, std::is_pointer, std::is_array, is_std_array, std::is_enum>::value && - (!std::is_reference<T>::value || - (std::is_lvalue_reference<T>::value && std::is_const<call_type>::value)); + static constexpr bool vectorize + = satisfies_any_of<call_type, std::is_arithmetic, is_complex, is_pod>::value + && satisfies_none_of<call_type, + std::is_pointer, + std::is_array, + is_std_array, + std::is_enum>::value + && (!std::is_reference<T>::value + || (std::is_lvalue_reference<T>::value && std::is_const<call_type>::value)); // Accept this type: an array for vectorized types, otherwise the type as-is: using type = conditional_t<vectorize, array_t<remove_cv_t<call_type>, array::forcecast>, T>; }; - // py::vectorize when a return type is present template <typename Func, typename Return, typename... Args> struct vectorize_returned_array { using Type = array_t<Return>; static Type create(broadcast_trivial trivial, const std::vector<ssize_t> &shape) { - if (trivial == broadcast_trivial::f_trivial) + if (trivial == broadcast_trivial::f_trivial) { return array_t<Return, array::f_style>(shape); - else - return array_t<Return>(shape); + } + return array_t<Return>(shape); } - static Return *mutable_data(Type &array) { - return array.mutable_data(); - } + static Return *mutable_data(Type &array) { return array.mutable_data(); } - static Return call(Func &f, Args &... args) { - return f(args...); - } + static Return call(Func &f, Args &...args) { return f(args...); } - static void call(Return *out, size_t i, Func &f, Args &... args) { - out[i] = f(args...); - } + static void call(Return *out, size_t i, Func &f, Args &...args) { out[i] = f(args...); } }; // py::vectorize when a return type is not present @@ -1501,25 +1780,18 @@ template <typename Func, typename... Args> struct vectorize_returned_array<Func, void, Args...> { using Type = none; - static Type create(broadcast_trivial, const std::vector<ssize_t> &) { - return none(); - } + static Type create(broadcast_trivial, const std::vector<ssize_t> &) { return none(); } - static void *mutable_data(Type &) { - return nullptr; - } + static void *mutable_data(Type &) { return nullptr; } - static detail::void_type call(Func &f, Args &... args) { + static detail::void_type call(Func &f, Args &...args) { f(args...); return {}; } - static void call(void *, size_t, Func &f, Args &... args) { - f(args...); - } + static void call(void *, size_t, Func &f, Args &...args) { f(args...); } }; - template <typename Func, typename Return, typename... Args> struct vectorize_helper { @@ -1532,12 +1804,16 @@ private: static constexpr size_t N = sizeof...(Args); static constexpr size_t NVectorized = constexpr_sum(vectorize_arg<Args>::vectorize...); - static_assert(NVectorized >= 1, - "pybind11::vectorize(...) requires a function with at least one vectorizable argument"); + static_assert( + NVectorized >= 1, + "pybind11::vectorize(...) requires a function with at least one vectorizable argument"); public: - template <typename T> - explicit vectorize_helper(T &&f) : f(std::forward<T>(f)) { } + template <typename T, + // SFINAE to prevent shadowing the copy constructor. + typename = detail::enable_if_t< + !std::is_same<vectorize_helper, typename std::decay<T>::type>::value>> + explicit vectorize_helper(T &&f) : f(std::forward<T>(f)) {} object operator()(typename vectorize_arg<Args>::type... args) { return run(args..., @@ -1549,10 +1825,11 @@ public: private: remove_reference_t<Func> f; - // Internal compiler error in MSVC 19.16.27025.1 (Visual Studio 2017 15.9.4), when compiling with "/permissive-" flag - // when arg_call_types is manually inlined. + // Internal compiler error in MSVC 19.16.27025.1 (Visual Studio 2017 15.9.4), when compiling + // with "/permissive-" flag when arg_call_types is manually inlined. using arg_call_types = std::tuple<typename vectorize_arg<Args>::call_type...>; - template <size_t Index> using param_n_t = typename std::tuple_element<Index, arg_call_types>::type; + template <size_t Index> + using param_n_t = typename std::tuple_element<Index, arg_call_types>::type; using returned_array = vectorize_returned_array<Func, Return, Args...>; @@ -1563,17 +1840,20 @@ private: // - BIndex is a incremental sequence (beginning at 0) of the same size as VIndex, so that // we can store vectorized buffer_infos in an array (argument VIndex has its buffer at // index BIndex in the array). - template <size_t... Index, size_t... VIndex, size_t... BIndex> object run( - typename vectorize_arg<Args>::type &...args, - index_sequence<Index...> i_seq, index_sequence<VIndex...> vi_seq, index_sequence<BIndex...> bi_seq) { + template <size_t... Index, size_t... VIndex, size_t... BIndex> + object run(typename vectorize_arg<Args>::type &...args, + index_sequence<Index...> i_seq, + index_sequence<VIndex...> vi_seq, + index_sequence<BIndex...> bi_seq) { // Pointers to values the function was called with; the vectorized ones set here will start // out as array_t<T> pointers, but they will be changed them to T pointers before we make // call the wrapped function. Non-vectorized pointers are left as-is. - std::array<void *, N> params{{ &args... }}; + std::array<void *, N> params{{&args...}}; // The array of `buffer_info`s of vectorized arguments: - std::array<buffer_info, NVectorized> buffers{{ reinterpret_cast<array *>(params[VIndex])->request()... }}; + std::array<buffer_info, NVectorized> buffers{ + {reinterpret_cast<array *>(params[VIndex])->request()...}}; /* Determine dimensions parameters of output array */ ssize_t nd = 0; @@ -1581,27 +1861,38 @@ private: auto trivial = broadcast(buffers, nd, shape); auto ndim = (size_t) nd; - size_t size = std::accumulate(shape.begin(), shape.end(), (size_t) 1, std::multiplies<size_t>()); + size_t size + = std::accumulate(shape.begin(), shape.end(), (size_t) 1, std::multiplies<size_t>()); // If all arguments are 0-dimension arrays (i.e. single values) return a plain value (i.e. // not wrapped in an array). if (size == 1 && ndim == 0) { PYBIND11_EXPAND_SIDE_EFFECTS(params[VIndex] = buffers[BIndex].ptr); - return cast(returned_array::call(f, *reinterpret_cast<param_n_t<Index> *>(params[Index])...)); + return cast( + returned_array::call(f, *reinterpret_cast<param_n_t<Index> *>(params[Index])...)); } auto result = returned_array::create(trivial, shape); - if (size == 0) return std::move(result); + PYBIND11_WARNING_PUSH +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") +#endif + + if (size == 0) { + return result; + } /* Call the function */ - auto mutable_data = returned_array::mutable_data(result); - if (trivial == broadcast_trivial::non_trivial) + auto *mutable_data = returned_array::mutable_data(result); + if (trivial == broadcast_trivial::non_trivial) { apply_broadcast(buffers, params, mutable_data, size, shape, i_seq, vi_seq, bi_seq); - else + } else { apply_trivial(buffers, params, mutable_data, size, i_seq, vi_seq, bi_seq); + } - return std::move(result); + return result; + PYBIND11_WARNING_POP } template <size_t... Index, size_t... VIndex, size_t... BIndex> @@ -1609,21 +1900,24 @@ private: std::array<void *, N> ¶ms, Return *out, size_t size, - index_sequence<Index...>, index_sequence<VIndex...>, index_sequence<BIndex...>) { + index_sequence<Index...>, + index_sequence<VIndex...>, + index_sequence<BIndex...>) { // Initialize an array of mutable byte references and sizes with references set to the // appropriate pointer in `params`; as we iterate, we'll increment each pointer by its size // (except for singletons, which get an increment of 0). - std::array<std::pair<unsigned char *&, const size_t>, NVectorized> vecparams{{ - std::pair<unsigned char *&, const size_t>( - reinterpret_cast<unsigned char *&>(params[VIndex] = buffers[BIndex].ptr), - buffers[BIndex].size == 1 ? 0 : sizeof(param_n_t<VIndex>) - )... - }}; + std::array<std::pair<unsigned char *&, const size_t>, NVectorized> vecparams{ + {std::pair<unsigned char *&, const size_t>( + reinterpret_cast<unsigned char *&>(params[VIndex] = buffers[BIndex].ptr), + buffers[BIndex].size == 1 ? 0 : sizeof(param_n_t<VIndex>))...}}; for (size_t i = 0; i < size; ++i) { - returned_array::call(out, i, f, *reinterpret_cast<param_n_t<Index> *>(params[Index])...); - for (auto &x : vecparams) x.first += x.second; + returned_array::call( + out, i, f, *reinterpret_cast<param_n_t<Index> *>(params[Index])...); + for (auto &x : vecparams) { + x.first += x.second; + } } } @@ -1633,61 +1927,72 @@ private: Return *out, size_t size, const std::vector<ssize_t> &output_shape, - index_sequence<Index...>, index_sequence<VIndex...>, index_sequence<BIndex...>) { + index_sequence<Index...>, + index_sequence<VIndex...>, + index_sequence<BIndex...>) { multi_array_iterator<NVectorized> input_iter(buffers, output_shape); for (size_t i = 0; i < size; ++i, ++input_iter) { - PYBIND11_EXPAND_SIDE_EFFECTS(( - params[VIndex] = input_iter.template data<BIndex>() - )); - returned_array::call(out, i, f, *reinterpret_cast<param_n_t<Index> *>(std::get<Index>(params))...); + PYBIND11_EXPAND_SIDE_EFFECTS((params[VIndex] = input_iter.template data<BIndex>())); + returned_array::call( + out, i, f, *reinterpret_cast<param_n_t<Index> *>(std::get<Index>(params))...); } } }; template <typename Func, typename Return, typename... Args> -vectorize_helper<Func, Return, Args...> -vectorize_extractor(const Func &f, Return (*) (Args ...)) { +vectorize_helper<Func, Return, Args...> vectorize_extractor(const Func &f, Return (*)(Args...)) { return detail::vectorize_helper<Func, Return, Args...>(f); } -template <typename T, int Flags> struct handle_type_name<array_t<T, Flags>> { - static constexpr auto name = _("numpy.ndarray[") + npy_format_descriptor<T>::name + _("]"); +template <typename T, int Flags> +struct handle_type_name<array_t<T, Flags>> { + static constexpr auto name + = const_name("numpy.ndarray[") + npy_format_descriptor<T>::name + const_name("]"); }; PYBIND11_NAMESPACE_END(detail) // Vanilla pointer vectorizer: template <typename Return, typename... Args> -detail::vectorize_helper<Return (*)(Args...), Return, Args...> -vectorize(Return (*f) (Args ...)) { +detail::vectorize_helper<Return (*)(Args...), Return, Args...> vectorize(Return (*f)(Args...)) { return detail::vectorize_helper<Return (*)(Args...), Return, Args...>(f); } // lambda vectorizer: template <typename Func, detail::enable_if_t<detail::is_lambda<Func>::value, int> = 0> -auto vectorize(Func &&f) -> decltype( - detail::vectorize_extractor(std::forward<Func>(f), (detail::function_signature_t<Func> *) nullptr)) { - return detail::vectorize_extractor(std::forward<Func>(f), (detail::function_signature_t<Func> *) nullptr); +auto vectorize(Func &&f) + -> decltype(detail::vectorize_extractor(std::forward<Func>(f), + (detail::function_signature_t<Func> *) nullptr)) { + return detail::vectorize_extractor(std::forward<Func>(f), + (detail::function_signature_t<Func> *) nullptr); } // Vectorize a class method (non-const): -template <typename Return, typename Class, typename... Args, - typename Helper = detail::vectorize_helper<decltype(std::mem_fn(std::declval<Return (Class::*)(Args...)>())), Return, Class *, Args...>> +template <typename Return, + typename Class, + typename... Args, + typename Helper = detail::vectorize_helper< + decltype(std::mem_fn(std::declval<Return (Class::*)(Args...)>())), + Return, + Class *, + Args...>> Helper vectorize(Return (Class::*f)(Args...)) { return Helper(std::mem_fn(f)); } // Vectorize a class method (const): -template <typename Return, typename Class, typename... Args, - typename Helper = detail::vectorize_helper<decltype(std::mem_fn(std::declval<Return (Class::*)(Args...) const>())), Return, const Class *, Args...>> +template <typename Return, + typename Class, + typename... Args, + typename Helper = detail::vectorize_helper< + decltype(std::mem_fn(std::declval<Return (Class::*)(Args...) const>())), + Return, + const Class *, + Args...>> Helper vectorize(Return (Class::*f)(Args...) const) { return Helper(std::mem_fn(f)); } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif diff --git a/include/pybind11/operators.h b/include/pybind11/operators.h index 086cb4cf..16a88ae1 100644 --- a/include/pybind11/operators.h +++ b/include/pybind11/operators.h @@ -11,24 +11,55 @@ #include "pybind11.h" -#if defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma clang diagnostic ignored "-Wunsequenced" // multiple unsequenced modifications to 'self' (when using def(py::self OP Type())) -#elif defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant -#endif - PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) /// Enumeration with all supported operator types enum op_id : int { - op_add, op_sub, op_mul, op_div, op_mod, op_divmod, op_pow, op_lshift, - op_rshift, op_and, op_xor, op_or, op_neg, op_pos, op_abs, op_invert, - op_int, op_long, op_float, op_str, op_cmp, op_gt, op_ge, op_lt, op_le, - op_eq, op_ne, op_iadd, op_isub, op_imul, op_idiv, op_imod, op_ilshift, - op_irshift, op_iand, op_ixor, op_ior, op_complex, op_bool, op_nonzero, - op_repr, op_truediv, op_itruediv, op_hash + op_add, + op_sub, + op_mul, + op_div, + op_mod, + op_divmod, + op_pow, + op_lshift, + op_rshift, + op_and, + op_xor, + op_or, + op_neg, + op_pos, + op_abs, + op_invert, + op_int, + op_long, + op_float, + op_str, + op_cmp, + op_gt, + op_ge, + op_lt, + op_le, + op_eq, + op_ne, + op_iadd, + op_isub, + op_imul, + op_idiv, + op_imod, + op_ilshift, + op_irshift, + op_iand, + op_ixor, + op_ior, + op_complex, + op_bool, + op_nonzero, + op_repr, + op_truediv, + op_itruediv, + op_hash }; enum op_type : int { @@ -37,125 +68,127 @@ enum op_type : int { op_u /* unary operator */ }; -struct self_t { }; +struct self_t {}; static const self_t self = self_t(); /// Type for an unused type slot -struct undefined_t { }; +struct undefined_t {}; /// Don't warn about an unused variable inline self_t __self() { return self; } /// base template of operator implementations -template <op_id, op_type, typename B, typename L, typename R> struct op_impl { }; +template <op_id, op_type, typename B, typename L, typename R> +struct op_impl {}; /// Operator implementation generator -template <op_id id, op_type ot, typename L, typename R> struct op_ { - template <typename Class, typename... Extra> void execute(Class &cl, const Extra&... extra) const { +template <op_id id, op_type ot, typename L, typename R> +struct op_ { + static constexpr bool op_enable_if_hook = true; + template <typename Class, typename... Extra> + void execute(Class &cl, const Extra &...extra) const { using Base = typename Class::type; using L_type = conditional_t<std::is_same<L, self_t>::value, Base, L>; using R_type = conditional_t<std::is_same<R, self_t>::value, Base, R>; using op = op_impl<id, ot, Base, L_type, R_type>; cl.def(op::name(), &op::execute, is_operator(), extra...); - #if PY_MAJOR_VERSION < 3 - if (id == op_truediv || id == op_itruediv) - cl.def(id == op_itruediv ? "__idiv__" : ot == op_l ? "__div__" : "__rdiv__", - &op::execute, is_operator(), extra...); - #endif } - template <typename Class, typename... Extra> void execute_cast(Class &cl, const Extra&... extra) const { + template <typename Class, typename... Extra> + void execute_cast(Class &cl, const Extra &...extra) const { using Base = typename Class::type; using L_type = conditional_t<std::is_same<L, self_t>::value, Base, L>; using R_type = conditional_t<std::is_same<R, self_t>::value, Base, R>; using op = op_impl<id, ot, Base, L_type, R_type>; cl.def(op::name(), &op::execute_cast, is_operator(), extra...); - #if PY_MAJOR_VERSION < 3 - if (id == op_truediv || id == op_itruediv) - cl.def(id == op_itruediv ? "__idiv__" : ot == op_l ? "__div__" : "__rdiv__", - &op::execute, is_operator(), extra...); - #endif } }; -#define PYBIND11_BINARY_OPERATOR(id, rid, op, expr) \ -template <typename B, typename L, typename R> struct op_impl<op_##id, op_l, B, L, R> { \ - static char const* name() { return "__" #id "__"; } \ - static auto execute(const L &l, const R &r) -> decltype(expr) { return (expr); } \ - static B execute_cast(const L &l, const R &r) { return B(expr); } \ -}; \ -template <typename B, typename L, typename R> struct op_impl<op_##id, op_r, B, L, R> { \ - static char const* name() { return "__" #rid "__"; } \ - static auto execute(const R &r, const L &l) -> decltype(expr) { return (expr); } \ - static B execute_cast(const R &r, const L &l) { return B(expr); } \ -}; \ -inline op_<op_##id, op_l, self_t, self_t> op(const self_t &, const self_t &) { \ - return op_<op_##id, op_l, self_t, self_t>(); \ -} \ -template <typename T> op_<op_##id, op_l, self_t, T> op(const self_t &, const T &) { \ - return op_<op_##id, op_l, self_t, T>(); \ -} \ -template <typename T> op_<op_##id, op_r, T, self_t> op(const T &, const self_t &) { \ - return op_<op_##id, op_r, T, self_t>(); \ -} - -#define PYBIND11_INPLACE_OPERATOR(id, op, expr) \ -template <typename B, typename L, typename R> struct op_impl<op_##id, op_l, B, L, R> { \ - static char const* name() { return "__" #id "__"; } \ - static auto execute(L &l, const R &r) -> decltype(expr) { return expr; } \ - static B execute_cast(L &l, const R &r) { return B(expr); } \ -}; \ -template <typename T> op_<op_##id, op_l, self_t, T> op(const self_t &, const T &) { \ - return op_<op_##id, op_l, self_t, T>(); \ -} - -#define PYBIND11_UNARY_OPERATOR(id, op, expr) \ -template <typename B, typename L> struct op_impl<op_##id, op_u, B, L, undefined_t> { \ - static char const* name() { return "__" #id "__"; } \ - static auto execute(const L &l) -> decltype(expr) { return expr; } \ - static B execute_cast(const L &l) { return B(expr); } \ -}; \ -inline op_<op_##id, op_u, self_t, undefined_t> op(const self_t &) { \ - return op_<op_##id, op_u, self_t, undefined_t>(); \ -} - -PYBIND11_BINARY_OPERATOR(sub, rsub, operator-, l - r) -PYBIND11_BINARY_OPERATOR(add, radd, operator+, l + r) -PYBIND11_BINARY_OPERATOR(mul, rmul, operator*, l * r) -PYBIND11_BINARY_OPERATOR(truediv, rtruediv, operator/, l / r) -PYBIND11_BINARY_OPERATOR(mod, rmod, operator%, l % r) -PYBIND11_BINARY_OPERATOR(lshift, rlshift, operator<<, l << r) -PYBIND11_BINARY_OPERATOR(rshift, rrshift, operator>>, l >> r) -PYBIND11_BINARY_OPERATOR(and, rand, operator&, l & r) -PYBIND11_BINARY_OPERATOR(xor, rxor, operator^, l ^ r) -PYBIND11_BINARY_OPERATOR(eq, eq, operator==, l == r) -PYBIND11_BINARY_OPERATOR(ne, ne, operator!=, l != r) -PYBIND11_BINARY_OPERATOR(or, ror, operator|, l | r) -PYBIND11_BINARY_OPERATOR(gt, lt, operator>, l > r) -PYBIND11_BINARY_OPERATOR(ge, le, operator>=, l >= r) -PYBIND11_BINARY_OPERATOR(lt, gt, operator<, l < r) -PYBIND11_BINARY_OPERATOR(le, ge, operator<=, l <= r) -//PYBIND11_BINARY_OPERATOR(pow, rpow, pow, std::pow(l, r)) -PYBIND11_INPLACE_OPERATOR(iadd, operator+=, l += r) -PYBIND11_INPLACE_OPERATOR(isub, operator-=, l -= r) -PYBIND11_INPLACE_OPERATOR(imul, operator*=, l *= r) -PYBIND11_INPLACE_OPERATOR(itruediv, operator/=, l /= r) -PYBIND11_INPLACE_OPERATOR(imod, operator%=, l %= r) -PYBIND11_INPLACE_OPERATOR(ilshift, operator<<=, l <<= r) -PYBIND11_INPLACE_OPERATOR(irshift, operator>>=, l >>= r) -PYBIND11_INPLACE_OPERATOR(iand, operator&=, l &= r) -PYBIND11_INPLACE_OPERATOR(ixor, operator^=, l ^= r) -PYBIND11_INPLACE_OPERATOR(ior, operator|=, l |= r) -PYBIND11_UNARY_OPERATOR(neg, operator-, -l) -PYBIND11_UNARY_OPERATOR(pos, operator+, +l) +#define PYBIND11_BINARY_OPERATOR(id, rid, op, expr) \ + template <typename B, typename L, typename R> \ + struct op_impl<op_##id, op_l, B, L, R> { \ + static char const *name() { return "__" #id "__"; } \ + static auto execute(const L &l, const R &r) -> decltype(expr) { return (expr); } \ + static B execute_cast(const L &l, const R &r) { return B(expr); } \ + }; \ + template <typename B, typename L, typename R> \ + struct op_impl<op_##id, op_r, B, L, R> { \ + static char const *name() { return "__" #rid "__"; } \ + static auto execute(const R &r, const L &l) -> decltype(expr) { return (expr); } \ + static B execute_cast(const R &r, const L &l) { return B(expr); } \ + }; \ + inline op_<op_##id, op_l, self_t, self_t> op(const self_t &, const self_t &) { \ + return op_<op_##id, op_l, self_t, self_t>(); \ + } \ + template <typename T> \ + op_<op_##id, op_l, self_t, T> op(const self_t &, const T &) { \ + return op_<op_##id, op_l, self_t, T>(); \ + } \ + template <typename T> \ + op_<op_##id, op_r, T, self_t> op(const T &, const self_t &) { \ + return op_<op_##id, op_r, T, self_t>(); \ + } + +#define PYBIND11_INPLACE_OPERATOR(id, op, expr) \ + template <typename B, typename L, typename R> \ + struct op_impl<op_##id, op_l, B, L, R> { \ + static char const *name() { return "__" #id "__"; } \ + static auto execute(L &l, const R &r) -> decltype(expr) { return expr; } \ + static B execute_cast(L &l, const R &r) { return B(expr); } \ + }; \ + template <typename T> \ + op_<op_##id, op_l, self_t, T> op(const self_t &, const T &) { \ + return op_<op_##id, op_l, self_t, T>(); \ + } + +#define PYBIND11_UNARY_OPERATOR(id, op, expr) \ + template <typename B, typename L> \ + struct op_impl<op_##id, op_u, B, L, undefined_t> { \ + static char const *name() { return "__" #id "__"; } \ + static auto execute(const L &l) -> decltype(expr) { return expr; } \ + static B execute_cast(const L &l) { return B(expr); } \ + }; \ + inline op_<op_##id, op_u, self_t, undefined_t> op(const self_t &) { \ + return op_<op_##id, op_u, self_t, undefined_t>(); \ + } + +PYBIND11_BINARY_OPERATOR(sub, rsub, operator-, l - r) +PYBIND11_BINARY_OPERATOR(add, radd, operator+, l + r) +PYBIND11_BINARY_OPERATOR(mul, rmul, operator*, l *r) +PYBIND11_BINARY_OPERATOR(truediv, rtruediv, operator/, l / r) +PYBIND11_BINARY_OPERATOR(mod, rmod, operator%, l % r) +PYBIND11_BINARY_OPERATOR(lshift, rlshift, operator<<, l << r) +PYBIND11_BINARY_OPERATOR(rshift, rrshift, operator>>, l >> r) +PYBIND11_BINARY_OPERATOR(and, rand, operator&, l &r) +PYBIND11_BINARY_OPERATOR(xor, rxor, operator^, l ^ r) +PYBIND11_BINARY_OPERATOR(eq, eq, operator==, l == r) +PYBIND11_BINARY_OPERATOR(ne, ne, operator!=, l != r) +PYBIND11_BINARY_OPERATOR(or, ror, operator|, l | r) +PYBIND11_BINARY_OPERATOR(gt, lt, operator>, l > r) +PYBIND11_BINARY_OPERATOR(ge, le, operator>=, l >= r) +PYBIND11_BINARY_OPERATOR(lt, gt, operator<, l < r) +PYBIND11_BINARY_OPERATOR(le, ge, operator<=, l <= r) +// PYBIND11_BINARY_OPERATOR(pow, rpow, pow, std::pow(l, r)) +PYBIND11_INPLACE_OPERATOR(iadd, operator+=, l += r) +PYBIND11_INPLACE_OPERATOR(isub, operator-=, l -= r) +PYBIND11_INPLACE_OPERATOR(imul, operator*=, l *= r) +PYBIND11_INPLACE_OPERATOR(itruediv, operator/=, l /= r) +PYBIND11_INPLACE_OPERATOR(imod, operator%=, l %= r) +PYBIND11_INPLACE_OPERATOR(ilshift, operator<<=, l <<= r) +PYBIND11_INPLACE_OPERATOR(irshift, operator>>=, l >>= r) +PYBIND11_INPLACE_OPERATOR(iand, operator&=, l &= r) +PYBIND11_INPLACE_OPERATOR(ixor, operator^=, l ^= r) +PYBIND11_INPLACE_OPERATOR(ior, operator|=, l |= r) +PYBIND11_UNARY_OPERATOR(neg, operator-, -l) +PYBIND11_UNARY_OPERATOR(pos, operator+, +l) // WARNING: This usage of `abs` should only be done for existing STL overloads. // Adding overloads directly in to the `std::` namespace is advised against: // https://en.cppreference.com/w/cpp/language/extending_std -PYBIND11_UNARY_OPERATOR(abs, abs, std::abs(l)) -PYBIND11_UNARY_OPERATOR(hash, hash, std::hash<L>()(l)) -PYBIND11_UNARY_OPERATOR(invert, operator~, (~l)) -PYBIND11_UNARY_OPERATOR(bool, operator!, !!l) -PYBIND11_UNARY_OPERATOR(int, int_, (int) l) -PYBIND11_UNARY_OPERATOR(float, float_, (double) l) +PYBIND11_UNARY_OPERATOR(abs, abs, std::abs(l)) +PYBIND11_UNARY_OPERATOR(hash, hash, std::hash<L>()(l)) +PYBIND11_UNARY_OPERATOR(invert, operator~, (~l)) +PYBIND11_UNARY_OPERATOR(bool, operator!, !!l) +PYBIND11_UNARY_OPERATOR(int, int_, (int) l) +PYBIND11_UNARY_OPERATOR(float, float_, (double) l) #undef PYBIND11_BINARY_OPERATOR #undef PYBIND11_INPLACE_OPERATOR @@ -167,7 +200,3 @@ using detail::self; using detail::hash; PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif diff --git a/include/pybind11/options.h b/include/pybind11/options.h index d74db1c6..1b212252 100644 --- a/include/pybind11/options.h +++ b/include/pybind11/options.h @@ -15,43 +15,70 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) class options { public: - // Default RAII constructor, which leaves settings as they currently are. options() : previous_state(global_state()) {} // Class is non-copyable. - options(const options&) = delete; - options& operator=(const options&) = delete; + options(const options &) = delete; + options &operator=(const options &) = delete; // Destructor, which restores settings that were in effect before. - ~options() { - global_state() = previous_state; - } + ~options() { global_state() = previous_state; } // Setter methods (affect the global state): - options& disable_user_defined_docstrings() & { global_state().show_user_defined_docstrings = false; return *this; } + options &disable_user_defined_docstrings() & { + global_state().show_user_defined_docstrings = false; + return *this; + } + + options &enable_user_defined_docstrings() & { + global_state().show_user_defined_docstrings = true; + return *this; + } + + options &disable_function_signatures() & { + global_state().show_function_signatures = false; + return *this; + } - options& enable_user_defined_docstrings() & { global_state().show_user_defined_docstrings = true; return *this; } + options &enable_function_signatures() & { + global_state().show_function_signatures = true; + return *this; + } - options& disable_function_signatures() & { global_state().show_function_signatures = false; return *this; } + options &disable_enum_members_docstring() & { + global_state().show_enum_members_docstring = false; + return *this; + } - options& enable_function_signatures() & { global_state().show_function_signatures = true; return *this; } + options &enable_enum_members_docstring() & { + global_state().show_enum_members_docstring = true; + return *this; + } // Getter methods (return the global state): - static bool show_user_defined_docstrings() { return global_state().show_user_defined_docstrings; } + static bool show_user_defined_docstrings() { + return global_state().show_user_defined_docstrings; + } static bool show_function_signatures() { return global_state().show_function_signatures; } + static bool show_enum_members_docstring() { + return global_state().show_enum_members_docstring; + } + // This type is not meant to be allocated on the heap. - void* operator new(size_t) = delete; + void *operator new(size_t) = delete; private: - struct state { - bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings. - bool show_function_signatures = true; //< Include auto-generated function signatures in docstrings. + bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings. + bool show_function_signatures = true; //< Include auto-generated function signatures + // in docstrings. + bool show_enum_members_docstring = true; //< Include auto-generated member list in enum + // docstrings. }; static state &global_state() { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3bffbb28..3bce1a01 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -10,104 +10,141 @@ #pragma once -#if defined(__INTEL_COMPILER) -# pragma warning push -# pragma warning disable 68 // integer conversion resulted in a change of sign -# pragma warning disable 186 // pointless comparison of unsigned integer with zero -# pragma warning disable 878 // incompatible exception specifications -# pragma warning disable 1334 // the "template" keyword used for syntactic disambiguation may only be used within a template -# pragma warning disable 1682 // implicit conversion of a 64-bit integral type to a smaller integral type (potential portability problem) -# pragma warning disable 1786 // function "strdup" was declared deprecated -# pragma warning disable 1875 // offsetof applied to non-POD (Plain Old Data) types is nonstandard -# pragma warning disable 2196 // warning #2196: routine is both "inline" and "noinline" -#elif defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4100) // warning C4100: Unreferenced formal parameter -# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant -# pragma warning(disable: 4512) // warning C4512: Assignment operator was implicitly defined as deleted -# pragma warning(disable: 4800) // warning C4800: 'int': forcing value to bool 'true' or 'false' (performance warning) -# pragma warning(disable: 4996) // warning C4996: The POSIX name for this item is deprecated. Instead, use the ISO C and C++ conformant name -# pragma warning(disable: 4702) // warning C4702: unreachable code -# pragma warning(disable: 4522) // warning C4522: multiple assignment operators specified -# pragma warning(disable: 4505) // warning C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -#elif defined(__GNUG__) && !defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-but-set-parameter" -# pragma GCC diagnostic ignored "-Wunused-but-set-variable" -# pragma GCC diagnostic ignored "-Wmissing-field-initializers" -# pragma GCC diagnostic ignored "-Wstrict-aliasing" -# pragma GCC diagnostic ignored "-Wattributes" -# if __GNUC__ >= 7 -# pragma GCC diagnostic ignored "-Wnoexcept-type" -# endif -#endif - -#include "attr.h" -#include "options.h" #include "detail/class.h" #include "detail/init.h" +#include "attr.h" +#include "gil.h" +#include "options.h" +#include <cstdlib> +#include <cstring> #include <memory> -#include <vector> +#include <new> #include <string> #include <utility> +#include <vector> +#if defined(__cpp_lib_launder) && !(defined(_MSC_VER) && (_MSC_VER < 1914)) +# define PYBIND11_STD_LAUNDER std::launder +# define PYBIND11_HAS_STD_LAUNDER 1 +#else +# define PYBIND11_STD_LAUNDER +# define PYBIND11_HAS_STD_LAUNDER 0 +#endif #if defined(__GNUG__) && !defined(__clang__) -# include <cxxabi.h> +# include <cxxabi.h> #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +/* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning + This warning is about ABI compatibility, not code health. + It is only actually needed in a couple places, but apparently GCC 7 "generates this warning if + and only if the first template instantiation ... involves noexcept" [stackoverflow], therefore + it could get triggered from seemingly random places, depending on user code. + No other GCC version generates this warning. + */ +#if defined(__GNUC__) && __GNUC__ == 7 +PYBIND11_WARNING_DISABLE_GCC("-Wnoexcept-type") +#endif + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +PYBIND11_NAMESPACE_BEGIN(detail) + +// Apply all the extensions translators from a list +// Return true if one of the translators completed without raising an exception +// itself. Return of false indicates that if there are other translators +// available, they should be tried. +inline bool apply_exception_translators(std::forward_list<ExceptionTranslator> &translators) { + auto last_exception = std::current_exception(); + + for (auto &translator : translators) { + try { + translator(last_exception); + return true; + } catch (...) { + last_exception = std::current_exception(); + } + } + return false; +} + +#if defined(_MSC_VER) +# define PYBIND11_COMPAT_STRDUP _strdup +#else +# define PYBIND11_COMPAT_STRDUP strdup +#endif + +PYBIND11_NAMESPACE_END(detail) + /// Wraps an arbitrary C++ function/method/lambda function/.. into a callable Python object class cpp_function : public function { public: cpp_function() = default; - cpp_function(std::nullptr_t) { } + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template <typename Return, typename... Args, typename... Extra> - cpp_function(Return (*f)(Args...), const Extra&... extra) { + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (*f)(Args...), const Extra &...extra) { initialize(f, f, extra...); } /// Construct a cpp_function from a lambda function (possibly with internal state) - template <typename Func, typename... Extra, + template <typename Func, + typename... Extra, typename = detail::enable_if_t<detail::is_lambda<Func>::value>> - cpp_function(Func &&f, const Extra&... extra) { - initialize(std::forward<Func>(f), - (detail::function_signature_t<Func> *) nullptr, extra...); + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Func &&f, const Extra &...extra) { + initialize( + std::forward<Func>(f), (detail::function_signature_t<Func> *) nullptr, extra...); } /// Construct a cpp_function from a class method (non-const, no ref-qualifier) template <typename Return, typename Class, typename... Arg, typename... Extra> - cpp_function(Return (Class::*f)(Arg...), const Extra&... extra) { - initialize([f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); }, - (Return (*) (Class *, Arg...)) nullptr, extra...); + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...), const Extra &...extra) { + initialize( + [f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); }, + (Return(*)(Class *, Arg...)) nullptr, + extra...); } /// Construct a cpp_function from a class method (non-const, lvalue ref-qualifier) /// A copy of the overload for non-const functions without explicit ref-qualifier /// but with an added `&`. template <typename Return, typename Class, typename... Arg, typename... Extra> - cpp_function(Return (Class::*f)(Arg...)&, const Extra&... extra) { - initialize([f](Class *c, Arg... args) -> Return { return (c->*f)(args...); }, - (Return (*) (Class *, Arg...)) nullptr, extra...); + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) &, const Extra &...extra) { + initialize( + [f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); }, + (Return(*)(Class *, Arg...)) nullptr, + extra...); } /// Construct a cpp_function from a class method (const, no ref-qualifier) template <typename Return, typename Class, typename... Arg, typename... Extra> - cpp_function(Return (Class::*f)(Arg...) const, const Extra&... extra) { - initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); }, - (Return (*)(const Class *, Arg ...)) nullptr, extra...); + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) const, const Extra &...extra) { + initialize([f](const Class *c, + Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); }, + (Return(*)(const Class *, Arg...)) nullptr, + extra...); } /// Construct a cpp_function from a class method (const, lvalue ref-qualifier) /// A copy of the overload for const functions without explicit ref-qualifier /// but with an added `&`. template <typename Return, typename Class, typename... Arg, typename... Extra> - cpp_function(Return (Class::*f)(Arg...) const&, const Extra&... extra) { - initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(args...); }, - (Return (*)(const Class *, Arg ...)) nullptr, extra...); + // NOLINTNEXTLINE(google-explicit-constructor) + cpp_function(Return (Class::*f)(Arg...) const &, const Extra &...extra) { + initialize([f](const Class *c, + Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); }, + (Return(*)(const Class *, Arg...)) nullptr, + extra...); } /// Return the function name @@ -117,9 +154,10 @@ protected: struct InitializingFunctionRecordDeleter { // `destruct(function_record, false)`: `initialize_generic` copies strings and // takes care of cleaning up in case of exceptions. So pass `false` to `free_strings`. - void operator()(detail::function_record * rec) { destruct(rec, false); } + void operator()(detail::function_record *rec) { destruct(rec, false); } }; - using unique_function_record = std::unique_ptr<detail::function_record, InitializingFunctionRecordDeleter>; + using unique_function_record + = std::unique_ptr<detail::function_record, InitializingFunctionRecordDeleter>; /// Space optimization: don't inline this frequently instantiated fragment PYBIND11_NOINLINE unique_function_record make_function_record() { @@ -128,69 +166,95 @@ protected: /// Special internal constructor for functors, lambda functions, etc. template <typename Func, typename Return, typename... Args, typename... Extra> - void initialize(Func &&f, Return (*)(Args...), const Extra&... extra) { + void initialize(Func &&f, Return (*)(Args...), const Extra &...extra) { using namespace detail; - struct capture { remove_reference_t<Func> f; }; + struct capture { + remove_reference_t<Func> f; + }; - /* Store the function including any extra state it might have (e.g. a lambda capture object) */ + /* Store the function including any extra state it might have (e.g. a lambda capture + * object) */ // The unique_ptr makes sure nothing is leaked in case of an exception. auto unique_rec = make_function_record(); - auto rec = unique_rec.get(); + auto *rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ if (sizeof(capture) <= sizeof(rec->data)) { /* Without these pragmas, GCC warns that there might not be enough space to use the placement new operator. However, the 'if' statement above ensures that this is the case. */ -#if defined(__GNUG__) && !defined(__clang__) && __GNUC__ >= 6 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wplacement-new" + PYBIND11_WARNING_PUSH + +#if defined(__GNUG__) && __GNUC__ >= 6 + PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif - new ((capture *) &rec->data) capture { std::forward<Func>(f) }; -#if defined(__GNUG__) && !defined(__clang__) && __GNUC__ >= 6 -# pragma GCC diagnostic pop + + new ((capture *) &rec->data) capture{std::forward<Func>(f)}; + +#if !PYBIND11_HAS_STD_LAUNDER + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") #endif - if (!std::is_trivially_destructible<Func>::value) - rec->free_data = [](function_record *r) { ((capture *) &r->data)->~capture(); }; + + // UB without std::launder, but without breaking ABI and/or + // a significant refactoring it's "impossible" to solve. + if (!std::is_trivially_destructible<capture>::value) { + rec->free_data = [](function_record *r) { + auto data = PYBIND11_STD_LAUNDER((capture *) &r->data); + (void) data; + data->~capture(); + }; + } + PYBIND11_WARNING_POP } else { - rec->data[0] = new capture { std::forward<Func>(f) }; + rec->data[0] = new capture{std::forward<Func>(f)}; rec->free_data = [](function_record *r) { delete ((capture *) r->data[0]); }; } /* Type casters for the function arguments and return value */ using cast_in = argument_loader<Args...>; - using cast_out = make_caster< - conditional_t<std::is_void<Return>::value, void_type, Return> - >; + using cast_out + = make_caster<conditional_t<std::is_void<Return>::value, void_type, Return>>; - static_assert(expected_num_args<Extra...>(sizeof...(Args), cast_in::has_args, cast_in::has_kwargs), - "The number of argument annotations does not match the number of function arguments"); + static_assert( + expected_num_args<Extra...>( + sizeof...(Args), cast_in::args_pos >= 0, cast_in::has_kwargs), + "The number of argument annotations does not match the number of function arguments"); /* Dispatch code which converts function arguments and performs the actual function call */ rec->impl = [](function_call &call) -> handle { cast_in args_converter; /* Try to cast the function arguments into the C++ domain */ - if (!args_converter.load_args(call)) + if (!args_converter.load_args(call)) { return PYBIND11_TRY_NEXT_OVERLOAD; + } /* Invoke call policy pre-call hook */ process_attributes<Extra...>::precall(call); /* Get a pointer to the capture object */ - auto data = (sizeof(capture) <= sizeof(call.func.data) - ? &call.func.data : call.func.data[0]); + const auto *data = (sizeof(capture) <= sizeof(call.func.data) ? &call.func.data + : call.func.data[0]); auto *cap = const_cast<capture *>(reinterpret_cast<const capture *>(data)); /* Override policy for rvalues -- usually to enforce rvp::move on an rvalue */ - return_value_policy policy = return_value_policy_override<Return>::policy(call.func.policy); + return_value_policy policy + = return_value_policy_override<Return>::policy(call.func.policy); /* Function scope guard -- defaults to the compile-to-nothing `void_type` */ using Guard = extract_guard_t<Extra...>; /* Perform the function call */ - handle result = cast_out::cast( - std::move(args_converter).template call<Return, Guard>(cap->f), policy, call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call<Return, Guard>(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call<Return, Guard>(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes<Extra...>::postcall(call, result); @@ -198,118 +262,156 @@ protected: return result; }; + rec->nargs_pos = cast_in::args_pos >= 0 + ? static_cast<std::uint16_t>(cast_in::args_pos) + : sizeof...(Args) - cast_in::has_kwargs; // Will get reduced more if + // we have a kw_only + rec->has_args = cast_in::args_pos >= 0; + rec->has_kwargs = cast_in::has_kwargs; + /* Process any user-provided function attributes */ process_attributes<Extra...>::init(extra..., rec); { constexpr bool has_kw_only_args = any_of<std::is_same<kw_only, Extra>...>::value, has_pos_only_args = any_of<std::is_same<pos_only, Extra>...>::value, - has_args = any_of<std::is_same<args, Args>...>::value, has_arg_annotations = any_of<is_keyword<Extra>...>::value; - static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations"); - static_assert(has_arg_annotations || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings and aligning the annotations to the argument)"); - static_assert(!(has_args && has_kw_only_args), "py::kw_only cannot be combined with a py::args argument"); + static_assert(has_arg_annotations || !has_kw_only_args, + "py::kw_only requires the use of argument annotations"); + static_assert(has_arg_annotations || !has_pos_only_args, + "py::pos_only requires the use of argument annotations (for docstrings " + "and aligning the annotations to the argument)"); + + static_assert(constexpr_sum(is_kw_only<Extra>::value...) <= 1, + "py::kw_only may be specified only once"); + static_assert(constexpr_sum(is_pos_only<Extra>::value...) <= 1, + "py::pos_only may be specified only once"); + constexpr auto kw_only_pos = constexpr_first<is_kw_only, Extra...>(); + constexpr auto pos_only_pos = constexpr_first<is_pos_only, Extra...>(); + static_assert(!(has_kw_only_args && has_pos_only_args) || pos_only_pos < kw_only_pos, + "py::pos_only must come before py::kw_only"); } - /* Generate a readable signature describing the function's arguments and return value types */ - static constexpr auto signature = _("(") + cast_in::arg_names + _(") -> ") + cast_out::name; + /* Generate a readable signature describing the function's arguments and return + value types */ + static constexpr auto signature + = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name; PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); /* Register the function with Python from generic (non-templated) code */ // Pass on the ownership over the `unique_rec` to `initialize_generic`. `rec` stays valid. initialize_generic(std::move(unique_rec), signature.text, types.data(), sizeof...(Args)); - if (cast_in::has_args) rec->has_args = true; - if (cast_in::has_kwargs) rec->has_kwargs = true; - /* Stash some additional information used by an important optimization in 'functional.h' */ using FunctionType = Return (*)(Args...); - constexpr bool is_function_ptr = - std::is_convertible<Func, FunctionType>::value && - sizeof(capture) == sizeof(void *); + constexpr bool is_function_ptr + = std::is_convertible<Func, FunctionType>::value && sizeof(capture) == sizeof(void *); if (is_function_ptr) { rec->is_stateless = true; - rec->data[1] = const_cast<void *>(reinterpret_cast<const void *>(&typeid(FunctionType))); + rec->data[1] + = const_cast<void *>(reinterpret_cast<const void *>(&typeid(FunctionType))); } } - // Utility class that keeps track of all duplicated strings, and cleans them up in its destructor, - // unless they are released. Basically a RAII-solution to deal with exceptions along the way. + // Utility class that keeps track of all duplicated strings, and cleans them up in its + // destructor, unless they are released. Basically a RAII-solution to deal with exceptions + // along the way. class strdup_guard { public: + strdup_guard() = default; + strdup_guard(const strdup_guard &) = delete; + strdup_guard &operator=(const strdup_guard &) = delete; + ~strdup_guard() { - for (auto s : strings) + for (auto *s : strings) { std::free(s); + } } char *operator()(const char *s) { - auto t = strdup(s); + auto *t = PYBIND11_COMPAT_STRDUP(s); strings.push_back(t); return t; } - void release() { - strings.clear(); - } + void release() { strings.clear(); } + private: std::vector<char *> strings; }; /// Register a function call with Python (generic non-templated code goes here) - void initialize_generic(unique_function_record &&unique_rec, const char *text, - const std::type_info *const *types, size_t args) { + void initialize_generic(unique_function_record &&unique_rec, + const char *text, + const std::type_info *const *types, + size_t args) { // Do NOT receive `unique_rec` by value. If this function fails to move out the unique_ptr, - // we do not want this to destuct the pointer. `initialize` (the caller) still relies on the - // pointee being alive after this call. Only move out if a `capsule` is going to keep it alive. - auto rec = unique_rec.get(); + // we do not want this to destruct the pointer. `initialize` (the caller) still relies on + // the pointee being alive after this call. Only move out if a `capsule` is going to keep + // it alive. + auto *rec = unique_rec.get(); // Keep track of strdup'ed strings, and clean them up as long as the function's capsule // has not taken ownership yet (when `unique_rec.release()` is called). - // Note: This cannot easily be fixed by a `unique_ptr` with custom deleter, because the strings - // are only referenced before strdup'ing. So only *after* the following block could `destruct` - // safely be called, but even then, `repr` could still throw in the middle of copying all strings. + // Note: This cannot easily be fixed by a `unique_ptr` with custom deleter, because the + // strings are only referenced before strdup'ing. So only *after* the following block could + // `destruct` safely be called, but even then, `repr` could still throw in the middle of + // copying all strings. strdup_guard guarded_strdup; /* Create copies of all referenced C-style strings */ rec->name = guarded_strdup(rec->name ? rec->name : ""); - if (rec->doc) rec->doc = guarded_strdup(rec->doc); - for (auto &a: rec->args) { - if (a.name) + if (rec->doc) { + rec->doc = guarded_strdup(rec->doc); + } + for (auto &a : rec->args) { + if (a.name) { a.name = guarded_strdup(a.name); - if (a.descr) + } + if (a.descr) { a.descr = guarded_strdup(a.descr); - else if (a.value) + } else if (a.value) { a.descr = guarded_strdup(repr(a.value).cast<std::string>().c_str()); + } } - rec->is_constructor = !strcmp(rec->name, "__init__") || !strcmp(rec->name, "__setstate__"); + rec->is_constructor = (std::strcmp(rec->name, "__init__") == 0) + || (std::strcmp(rec->name, "__setstate__") == 0); -#if !defined(NDEBUG) && !defined(PYBIND11_DISABLE_NEW_STYLE_INIT_WARNING) +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(PYBIND11_DISABLE_NEW_STYLE_INIT_WARNING) if (rec->is_constructor && !rec->is_new_style_constructor) { - const auto class_name = detail::get_fully_qualified_tp_name((PyTypeObject *) rec->scope.ptr()); + const auto class_name + = detail::get_fully_qualified_tp_name((PyTypeObject *) rec->scope.ptr()); const auto func_name = std::string(rec->name); - PyErr_WarnEx( - PyExc_FutureWarning, - ("pybind11-bound class '" + class_name + "' is using an old-style " - "placement-new '" + func_name + "' which has been deprecated. See " - "the upgrade guide in pybind11's docs. This message is only visible " - "when compiled in debug mode.").c_str(), 0 - ); + PyErr_WarnEx(PyExc_FutureWarning, + ("pybind11-bound class '" + class_name + + "' is using an old-style " + "placement-new '" + + func_name + + "' which has been deprecated. See " + "the upgrade guide in pybind11's docs. This message is only visible " + "when compiled in debug mode.") + .c_str(), + 0); } #endif /* Generate a proper function signature */ std::string signature; size_t type_index = 0, arg_index = 0; - for (auto *pc = text; *pc != '\0'; ++pc) { + bool is_starred = false; + for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; if (c == '{') { // Write arg name for everything except *args and **kwargs. - if (*(pc + 1) == '*') + is_starred = *(pc + 1) == '*'; + if (is_starred) { continue; + } // Separator for keyword-only arguments, placed before the kw - // arguments start - if (rec->nargs_kw_only > 0 && arg_index + rec->nargs_kw_only == args) + // arguments start (unless we are already putting an *args) + if (!rec->has_args && arg_index == rec->nargs_pos) { signature += "*, "; + } if (arg_index < rec->args.size() && rec->args[arg_index].name) { signature += rec->args[arg_index].name; } else if (arg_index == 0 && rec->is_method) { @@ -320,30 +422,32 @@ protected: signature += ": "; } else if (c == '}') { // Write default value if available. - if (arg_index < rec->args.size() && rec->args[arg_index].descr) { + if (!is_starred && arg_index < rec->args.size() && rec->args[arg_index].descr) { signature += " = "; signature += rec->args[arg_index].descr; } // Separator for positional-only arguments (placed after the // argument, rather than before like * - if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only) + if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only) { signature += ", /"; - arg_index++; + } + if (!is_starred) { + arg_index++; + } } else if (c == '%') { const std::type_info *t = types[type_index++]; - if (!t) + if (!t) { pybind11_fail("Internal error while parsing type signature (1)"); - if (auto tinfo = detail::get_type_info(*t)) { + } + if (auto *tinfo = detail::get_type_info(*t)) { handle th((PyObject *) tinfo->type); - signature += - th.attr("__module__").cast<std::string>() + "." + - th.attr("__qualname__").cast<std::string>(); // Python 3.3+, but we backport it to earlier versions + signature += th.attr("__module__").cast<std::string>() + "." + + th.attr("__qualname__").cast<std::string>(); } else if (rec->is_new_style_constructor && arg_index == 0) { // A new-style `__init__` takes `self` as `value_and_holder`. // Rewrite it to the proper class type. - signature += - rec->scope.attr("__module__").cast<std::string>() + "." + - rec->scope.attr("__qualname__").cast<std::string>(); + signature += rec->scope.attr("__module__").cast<std::string>() + "." + + rec->scope.attr("__qualname__").cast<std::string>(); } else { std::string tname(t->name()); detail::clean_type_id(tname); @@ -354,39 +458,44 @@ protected: } } - if (arg_index != args || types[type_index] != nullptr) + if (arg_index != args - rec->has_args - rec->has_kwargs || types[type_index] != nullptr) { pybind11_fail("Internal error while parsing type signature (2)"); - -#if PY_MAJOR_VERSION < 3 - if (strcmp(rec->name, "__next__") == 0) { - std::free(rec->name); - rec->name = guarded_strdup("next"); - } else if (strcmp(rec->name, "__bool__") == 0) { - std::free(rec->name); - rec->name = guarded_strdup("__nonzero__"); } -#endif + rec->signature = guarded_strdup(signature.c_str()); rec->args.shrink_to_fit(); rec->nargs = (std::uint16_t) args; - if (rec->sibling && PYBIND11_INSTANCE_METHOD_CHECK(rec->sibling.ptr())) + if (rec->sibling && PYBIND11_INSTANCE_METHOD_CHECK(rec->sibling.ptr())) { rec->sibling = PYBIND11_INSTANCE_METHOD_GET_FUNCTION(rec->sibling.ptr()); + } detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto rec_capsule = reinterpret_borrow<capsule>(PyCFunction_GET_SELF(rec->sibling.ptr())); - chain = (detail::function_record *) rec_capsule; - /* Never append a method to an overload chain of a parent class; - instead, hide the parent's overloads in this case */ - if (!chain->scope.is(rec->scope)) + auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); + if (!isinstance<capsule>(self)) { chain = nullptr; + } else { + auto rec_capsule = reinterpret_borrow<capsule>(self); + if (detail::is_function_record_capsule(rec_capsule)) { + chain = rec_capsule.get_pointer<detail::function_record>(); + /* Never append a method to an overload chain of a parent class; + instead, hide the parent's overloads in this case */ + if (!chain->scope.is(rec->scope)) { + chain = nullptr; + } + } else { + chain = nullptr; + } + } + } + // Don't trigger for things like the default __init__, which are wrapper_descriptors + // that we are intentionally replacing + else if (!rec->sibling.is_none() && rec->name[0] != '_') { + pybind11_fail("Cannot overload existing non-function object \"" + + std::string(rec->name) + "\" with a function of the same name"); } - // Don't trigger for things like the default __init__, which are wrapper_descriptors that we are intentionally replacing - else if (!rec->sibling.is_none() && rec->name[0] != '_') - pybind11_fail("Cannot overload existing non-function object \"" + std::string(rec->name) + - "\" with a function of the same name"); } if (!chain) { @@ -394,12 +503,13 @@ protected: rec->def = new PyMethodDef(); std::memset(rec->def, 0, sizeof(PyMethodDef)); rec->def->ml_name = rec->name; - rec->def->ml_meth = reinterpret_cast<PyCFunction>(reinterpret_cast<void (*) (void)>(*dispatcher)); + rec->def->ml_meth + = reinterpret_cast<PyCFunction>(reinterpret_cast<void (*)()>(dispatcher)); rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; - capsule rec_capsule(unique_rec.release(), [](void *ptr) { - destruct((detail::function_record *) ptr); - }); + capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), + [](void *ptr) { destruct((detail::function_record *) ptr); }); guarded_strdup.release(); object scope_module; @@ -412,21 +522,27 @@ protected: } m_ptr = PyCFunction_NewEx(rec->def, rec_capsule.ptr(), scope_module.ptr()); - if (!m_ptr) + if (!m_ptr) { pybind11_fail("cpp_function::cpp_function(): Could not allocate function object"); + } } else { /* Append at the beginning or end of the overload chain */ m_ptr = rec->sibling.ptr(); inc_ref(); - if (chain->is_method != rec->is_method) - pybind11_fail("overloading a method with both static and instance methods is not supported; " - #if defined(NDEBUG) - "compile in debug mode for more details" - #else - "error while attempting to bind " + std::string(rec->is_method ? "instance" : "static") + " method " + - std::string(pybind11::str(rec->scope.attr("__name__"))) + "." + std::string(rec->name) + signature - #endif + if (chain->is_method != rec->is_method) { + pybind11_fail( + "overloading a method with both static and instance methods is not supported; " +#if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more " + "details" +#else + "error while attempting to bind " + + std::string(rec->is_method ? "instance" : "static") + " method " + + std::string(pybind11::str(rec->scope.attr("__name__"))) + "." + + std::string(rec->name) + signature +#endif ); + } if (rec->prepend) { // Beginning of chain; we need to replace the capsule's current head-of-the-chain @@ -434,14 +550,16 @@ protected: // chain. chain_start = rec; rec->next = chain; - auto rec_capsule = reinterpret_borrow<capsule>(((PyCFunctionObject *) m_ptr)->m_self); + auto rec_capsule + = reinterpret_borrow<capsule>(((PyCFunctionObject *) m_ptr)->m_self); rec_capsule.set_pointer(unique_rec.release()); guarded_strdup.release(); } else { // Or end of chain (normal behavior) chain_start = chain; - while (chain->next) + while (chain->next) { chain = chain->next; + } chain->next = unique_rec.release(); guarded_strdup.release(); } @@ -459,25 +577,35 @@ protected: } // Then specific overload signatures bool first_user_def = true; - for (auto it = chain_start; it != nullptr; it = it->next) { + for (auto *it = chain_start; it != nullptr; it = it->next) { if (options::show_function_signatures()) { - if (index > 0) signatures += "\n"; - if (chain) + if (index > 0) { + signatures += '\n'; + } + if (chain) { signatures += std::to_string(++index) + ". "; + } signatures += rec->name; signatures += it->signature; - signatures += "\n"; + signatures += '\n'; } - if (it->doc && strlen(it->doc) > 0 && options::show_user_defined_docstrings()) { - // If we're appending another docstring, and aren't printing function signatures, we - // need to append a newline first: + if (it->doc && it->doc[0] != '\0' && options::show_user_defined_docstrings()) { + // If we're appending another docstring, and aren't printing function signatures, + // we need to append a newline first: if (!options::show_function_signatures()) { - if (first_user_def) first_user_def = false; - else signatures += "\n"; + if (first_user_def) { + first_user_def = false; + } else { + signatures += '\n'; + } + } + if (options::show_function_signatures()) { + signatures += '\n'; } - if (options::show_function_signatures()) signatures += "\n"; signatures += it->doc; - if (options::show_function_signatures()) signatures += "\n"; + if (options::show_function_signatures()) { + signatures += '\n'; + } } } @@ -485,28 +613,32 @@ protected: auto *func = (PyCFunctionObject *) m_ptr; std::free(const_cast<char *>(func->m_ml->ml_doc)); // Install docstring if it's non-empty (when at least one option is enabled) - func->m_ml->ml_doc = signatures.empty() ? nullptr : strdup(signatures.c_str()); + func->m_ml->ml_doc + = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); if (rec->is_method) { m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); - if (!m_ptr) - pybind11_fail("cpp_function::cpp_function(): Could not allocate instance method object"); + if (!m_ptr) { + pybind11_fail( + "cpp_function::cpp_function(): Could not allocate instance method object"); + } Py_DECREF(func); } } /// When a cpp_function is GCed, release any memory allocated by pybind11 static void destruct(detail::function_record *rec, bool free_strings = true) { - // If on Python 3.9, check the interpreter "MICRO" (patch) version. - // If this is running on 3.9.0, we have to work around a bug. - #if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 - static bool is_zero = Py_GetVersion()[4] == '0'; - #endif +// If on Python 3.9, check the interpreter "MICRO" (patch) version. +// If this is running on 3.9.0, we have to work around a bug. +#if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 + static bool is_zero = Py_GetVersion()[4] == '0'; +#endif while (rec) { detail::function_record *next = rec->next; - if (rec->free_data) + if (rec->free_data) { rec->free_data(rec); + } // During initialization, these strings might not have been copied yet, // so they cannot be freed. Once the function has been created, they can. // Check `make_function_record` for more details. @@ -514,24 +646,26 @@ protected: std::free((char *) rec->name); std::free((char *) rec->doc); std::free((char *) rec->signature); - for (auto &arg: rec->args) { + for (auto &arg : rec->args) { std::free(const_cast<char *>(arg.name)); std::free(const_cast<char *>(arg.descr)); } } - for (auto &arg: rec->args) + for (auto &arg : rec->args) { arg.value.dec_ref(); + } if (rec->def) { std::free(const_cast<char *>(rec->def->ml_doc)); - // Python 3.9.0 decref's these in the wrong order; rec->def - // If loaded on 3.9.0, let these leak (use Python 3.9.1 at runtime to fix) - // See https://github.com/python/cpython/pull/22670 - #if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 - if (!is_zero) - delete rec->def; - #else +// Python 3.9.0 decref's these in the wrong order; rec->def +// If loaded on 3.9.0, let these leak (use Python 3.9.1 at runtime to fix) +// See https://github.com/python/cpython/pull/22670 +#if !defined(PYPY_VERSION) && PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION == 9 + if (!is_zero) { delete rec->def; - #endif + } +#else + delete rec->def; +#endif } delete rec; rec = next; @@ -541,12 +675,16 @@ protected: /// Main dispatch logic for calls to functions bound using pybind11 static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { using namespace detail; + assert(isinstance<capsule>(self)); /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + const function_record *overloads = reinterpret_cast<function_record *>( + PyCapsule_GetPointer(self, get_function_record_capsule_name())), *it = overloads; + assert(overloads != nullptr); - /* Need to know how many arguments + keyword arguments there are to pick the right overload */ + /* Need to know how many arguments + keyword arguments there are to pick the right + overload */ const auto n_args_in = (size_t) PyTuple_GET_SIZE(args_in); handle parent = n_args_in > 0 ? PyTuple_GET_ITEM(args_in, 0) : nullptr, @@ -554,19 +692,23 @@ protected: auto self_value_and_holder = value_and_holder(); if (overloads->is_constructor) { - if (!PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { - PyErr_SetString(PyExc_TypeError, "__init__(self, ...) called with invalid `self` argument"); + if (!parent + || !PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { + PyErr_SetString( + PyExc_TypeError, + "__init__(self, ...) called with invalid or missing `self` argument"); return nullptr; } - const auto tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); - const auto pi = reinterpret_cast<instance *>(parent.ptr()); + auto *const tinfo = get_type_info((PyTypeObject *) overloads->scope.ptr()); + auto *const pi = reinterpret_cast<instance *>(parent.ptr()); self_value_and_holder = pi->get_value_and_holder(tinfo, true); // If this value is already registered it must mean __init__ is invoked multiple times; // we really can't support that in C++, so just ignore the second __init__. - if (self_value_and_holder.instance_registered()) + if (self_value_and_holder.instance_registered()) { return none().release().ptr(); + } } try { @@ -585,44 +727,53 @@ protected: 1. Copy all positional arguments we were given, also checking to make sure that named positional arguments weren't *also* specified via kwarg. 2. If we weren't given enough, try to make up the omitted ones by checking - whether they were provided by a kwarg matching the `py::arg("name")` name. If - so, use it (and remove it from kwargs; if not, see if the function binding + whether they were provided by a kwarg matching the `py::arg("name")` name. If + so, use it (and remove it from kwargs); if not, see if the function binding provided a default that we can use. - 3. Ensure that either all keyword arguments were "consumed", or that the function - takes a kwargs argument to accept unconsumed kwargs. + 3. Ensure that either all keyword arguments were "consumed", or that the + function takes a kwargs argument to accept unconsumed kwargs. 4. Any positional arguments still left get put into a tuple (for args), and any leftover kwargs get put into a dict. 5. Pack everything into a vector; if we have py::args or py::kwargs, they are an extra tuple or dict at the end of the positional arguments. 6. Call the function call dispatcher (function_record::impl) - If one of these fail, move on to the next overload and keep trying until we get a - result other than PYBIND11_TRY_NEXT_OVERLOAD. + If one of these fail, move on to the next overload and keep trying until we get + a result other than PYBIND11_TRY_NEXT_OVERLOAD. */ const function_record &func = *it; - size_t num_args = func.nargs; // Number of positional arguments that we need - if (func.has_args) --num_args; // (but don't count py::args - if (func.has_kwargs) --num_args; // or py::kwargs) - size_t pos_args = num_args - func.nargs_kw_only; + size_t num_args = func.nargs; // Number of positional arguments that we need + if (func.has_args) { + --num_args; // (but don't count py::args + } + if (func.has_kwargs) { + --num_args; // or py::kwargs) + } + size_t pos_args = func.nargs_pos; - if (!func.has_args && n_args_in > pos_args) + if (!func.has_args && n_args_in > pos_args) { continue; // Too many positional arguments for this overload + } - if (n_args_in < pos_args && func.args.size() < pos_args) - continue; // Not enough positional arguments given, and not enough defaults to fill in the blanks + if (n_args_in < pos_args && func.args.size() < pos_args) { + continue; // Not enough positional arguments given, and not enough defaults to + // fill in the blanks + } function_call call(func, parent); - size_t args_to_copy = (std::min)(pos_args, n_args_in); // Protect std::min with parentheses + // Protect std::min with parentheses + size_t args_to_copy = (std::min)(pos_args, n_args_in); size_t args_copied = 0; // 0. Inject new-style `self` argument if (func.is_new_style_constructor) { // The `value` may have been preallocated by an old-style `__init__` // if it was a preceding candidate for overload resolution. - if (self_value_and_holder) + if (self_value_and_holder) { self_value_and_holder.type->dealloc(self_value_and_holder); + } call.init_self = PyTuple_GET_ITEM(args_in, 0); call.args.emplace_back(reinterpret_cast<PyObject *>(&self_value_and_holder)); @@ -633,8 +784,10 @@ protected: // 1. Copy any position arguments given. bool bad_arg = false; for (; args_copied < args_to_copy; ++args_copied) { - const argument_record *arg_rec = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; - if (kwargs_in && arg_rec && arg_rec->name && PyDict_GetItemString(kwargs_in, arg_rec->name)) { + const argument_record *arg_rec + = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; + if (kwargs_in && arg_rec && arg_rec->name + && dict_getitemstring(kwargs_in, arg_rec->name)) { bad_arg = true; break; } @@ -647,8 +800,13 @@ protected: call.args.push_back(arg); call.args_convert.push_back(arg_rec ? arg_rec->convert : true); } - if (bad_arg) + if (bad_arg) { continue; // Maybe it was meant for another overload (issue #688) + } + + // Keep track of how many position args we copied out in case we need to come back + // to copy the rest into a py::args argument. + size_t positional_args_copied = args_copied; // We'll need to copy this if we steal some kwargs for defaults dict kwargs = reinterpret_borrow<dict>(kwargs_in); @@ -665,12 +823,14 @@ protected: if (value) { call.args.push_back(value); call.args_convert.push_back(arg_rec.convert); - } else + } else { break; + } } - if (args_copied < func.nargs_pos_only) + if (args_copied < func.nargs_pos_only) { continue; // Not enough defaults to fill the positional arguments + } } // 2. Check kwargs and, failing that, defaults that may help complete the list @@ -681,8 +841,9 @@ protected: const auto &arg_rec = func.args[args_copied]; handle value; - if (kwargs_in && arg_rec.name) - value = PyDict_GetItemString(kwargs.ptr(), arg_rec.name); + if (kwargs_in && arg_rec.name) { + value = dict_getitemstring(kwargs.ptr(), arg_rec.name); + } if (value) { // Consume a kwargs value @@ -690,7 +851,9 @@ protected: kwargs = reinterpret_steal<dict>(PyDict_Copy(kwargs.ptr())); copied_kwargs = true; } - PyDict_DelItemString(kwargs.ptr(), arg_rec.name); + if (PyDict_DelItemString(kwargs.ptr(), arg_rec.name) == -1) { + throw error_already_set(); + } } else if (arg_rec.value) { value = arg_rec.value; } @@ -700,20 +863,29 @@ protected: } if (value) { + // If we're at the py::args index then first insert a stub for it to be + // replaced later + if (func.has_args && call.args.size() == func.nargs_pos) { + call.args.push_back(none()); + } + call.args.push_back(value); call.args_convert.push_back(arg_rec.convert); - } - else + } else { break; + } } - if (args_copied < num_args) - continue; // Not enough arguments, defaults, or kwargs to fill the positional arguments + if (args_copied < num_args) { + continue; // Not enough arguments, defaults, or kwargs to fill the + // positional arguments + } } // 3. Check everything was consumed (unless we have a kwargs arg) - if (kwargs && !kwargs.empty() && !func.has_kwargs) + if (kwargs && !kwargs.empty() && !func.has_kwargs) { continue; // Unconsumed kwargs, but no py::kwargs argument to accept them + } // 4a. If we have a py::args argument, create a new tuple with leftovers if (func.has_args) { @@ -722,35 +894,42 @@ protected: // We didn't copy out any position arguments from the args_in tuple, so we // can reuse it directly without copying: extra_args = reinterpret_borrow<tuple>(args_in); - } else if (args_copied >= n_args_in) { + } else if (positional_args_copied >= n_args_in) { extra_args = tuple(0); } else { - size_t args_size = n_args_in - args_copied; + size_t args_size = n_args_in - positional_args_copied; extra_args = tuple(args_size); for (size_t i = 0; i < args_size; ++i) { - extra_args[i] = PyTuple_GET_ITEM(args_in, args_copied + i); + extra_args[i] = PyTuple_GET_ITEM(args_in, positional_args_copied + i); } } - call.args.push_back(extra_args); + if (call.args.size() <= func.nargs_pos) { + call.args.push_back(extra_args); + } else { + call.args[func.nargs_pos] = extra_args; + } call.args_convert.push_back(false); call.args_ref = std::move(extra_args); } // 4b. If we have a py::kwargs, pass on any remaining kwargs if (func.has_kwargs) { - if (!kwargs.ptr()) + if (!kwargs.ptr()) { kwargs = dict(); // If we didn't get one, send an empty one + } call.args.push_back(kwargs); call.args_convert.push_back(false); call.kwargs_ref = std::move(kwargs); } - // 5. Put everything in a vector. Not technically step 5, we've been building it - // in `call.args` all along. - #if !defined(NDEBUG) - if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) - pybind11_fail("Internal error: function call dispatcher inserted wrong number of arguments!"); - #endif +// 5. Put everything in a vector. Not technically step 5, we've been building it +// in `call.args` all along. +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) { + pybind11_fail("Internal error: function call dispatcher inserted wrong number " + "of arguments!"); + } +#endif std::vector<bool> second_pass_convert; if (overloaded) { @@ -769,8 +948,9 @@ protected: result = PYBIND11_TRY_NEXT_OVERLOAD; } - if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) + if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { break; + } if (overloaded) { // The (overloaded) call failed; if the call has at least one argument that @@ -789,7 +969,8 @@ protected: } if (overloaded && !second_pass.empty() && result.ptr() == PYBIND11_TRY_NEXT_OVERLOAD) { - // The no-conversion pass finished without success, try again with conversion allowed + // The no-conversion pass finished without success, try again with conversion + // allowed for (auto &call : second_pass) { try { loader_life_support guard{}; @@ -801,8 +982,9 @@ protected: if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { // The error reporting logic below expects 'it' to be valid, as it would be // if we'd encountered this failure in the first-pass loop. - if (!result) + if (!result) { it = &call.func; + } break; } } @@ -811,33 +993,38 @@ protected: e.restore(); return nullptr; #ifdef __GLIBCXX__ - } catch ( abi::__forced_unwind& ) { + } catch (abi::__forced_unwind &) { throw; #endif } catch (...) { /* When an exception is caught, give each registered exception - translator a chance to translate it to a Python exception - in reverse order of registration. + translator a chance to translate it to a Python exception. First + all module-local translators will be tried in reverse order of + registration. If none of the module-locale translators handle + the exception (or there are no module-locale translators) then + the global translators will be tried, also in reverse order of + registration. A translator may choose to do one of the following: - catch the exception and call PyErr_SetString or PyErr_SetObject to set a standard (or custom) Python exception, or - do nothing and let the exception fall through to the next translator, or - - delegate translation to the next translator by throwing a new type of exception. */ + - delegate translation to the next translator by throwing a new type of exception. + */ - auto last_exception = std::current_exception(); - auto ®istered_exception_translators = get_internals().registered_exception_translators; - for (auto& translator : registered_exception_translators) { - try { - translator(last_exception); - } catch (...) { - last_exception = std::current_exception(); - continue; - } + auto &local_exception_translators + = get_local_internals().registered_exception_translators; + if (detail::apply_exception_translators(local_exception_translators)) { return nullptr; } - PyErr_SetString(PyExc_SystemError, "Exception escaped from default exception translator!"); + auto &exception_translators = get_internals().registered_exception_translators; + if (detail::apply_exception_translators(exception_translators)) { + return nullptr; + } + + PyErr_SetString(PyExc_SystemError, + "Exception escaped from default exception translator!"); return nullptr; } @@ -852,20 +1039,22 @@ protected: }; if (result.ptr() == PYBIND11_TRY_NEXT_OVERLOAD) { - if (overloads->is_operator) + if (overloads->is_operator) { return handle(Py_NotImplemented).inc_ref().ptr(); + } - std::string msg = std::string(overloads->name) + "(): incompatible " + - std::string(overloads->is_constructor ? "constructor" : "function") + - " arguments. The following argument types are supported:\n"; + std::string msg = std::string(overloads->name) + "(): incompatible " + + std::string(overloads->is_constructor ? "constructor" : "function") + + " arguments. The following argument types are supported:\n"; int ctr = 0; for (const function_record *it2 = overloads; it2 != nullptr; it2 = it2->next) { - msg += " "+ std::to_string(++ctr) + ". "; + msg += " " + std::to_string(++ctr) + ". "; bool wrote_sig = false; if (overloads->is_constructor) { - // For a constructor, rewrite `(self: Object, arg0, ...) -> NoneType` as `Object(arg0, ...)` + // For a constructor, rewrite `(self: Object, arg0, ...) -> NoneType` as + // `Object(arg0, ...)` std::string sig = it2->signature; size_t start = sig.find('(') + 7; // skip "(self: " if (start < sig.size()) { @@ -873,7 +1062,9 @@ protected: size_t end = sig.find(", "), next = end + 2; size_t ret = sig.rfind(" -> "); // Or the ), if there is no comma: - if (end >= sig.size()) next = end = sig.find(')'); + if (end >= sig.size()) { + next = end = sig.find(')'); + } if (start < end && next < sig.size()) { msg.append(sig, start, end - start); msg += '('; @@ -882,35 +1073,45 @@ protected: } } } - if (!wrote_sig) msg += it2->signature; + if (!wrote_sig) { + msg += it2->signature; + } - msg += "\n"; + msg += '\n'; } msg += "\nInvoked with: "; auto args_ = reinterpret_borrow<tuple>(args_in); bool some_args = false; for (size_t ti = overloads->is_constructor ? 1 : 0; ti < args_.size(); ++ti) { - if (!some_args) some_args = true; - else msg += ", "; + if (!some_args) { + some_args = true; + } else { + msg += ", "; + } try { msg += pybind11::repr(args_[ti]); - } catch (const error_already_set&) { + } catch (const error_already_set &) { msg += "<repr raised Error>"; } } if (kwargs_in) { auto kwargs = reinterpret_borrow<dict>(kwargs_in); if (!kwargs.empty()) { - if (some_args) msg += "; "; + if (some_args) { + msg += "; "; + } msg += "kwargs: "; bool first = true; for (auto kwarg : kwargs) { - if (first) first = false; - else msg += ", "; + if (first) { + first = false; + } else { + msg += ", "; + } msg += pybind11::str("{}=").format(kwarg.first); try { msg += pybind11::repr(kwarg.second); - } catch (const error_already_set&) { + } catch (const error_already_set &) { msg += "<repr raised Error>"; } } @@ -918,22 +1119,33 @@ protected: } append_note_if_missing_header_is_suspected(msg); + // Attach additional error info to the exception if supported + if (PyErr_Occurred()) { + // #HelpAppreciated: unit test coverage for this branch. + raise_from(PyExc_TypeError, msg.c_str()); + return nullptr; + } PyErr_SetString(PyExc_TypeError, msg.c_str()); return nullptr; - } else if (!result) { + } + if (!result) { std::string msg = "Unable to convert function return value to a " "Python type! The signature was\n\t"; msg += it->signature; append_note_if_missing_header_is_suspected(msg); + // Attach additional error info to the exception if supported + if (PyErr_Occurred()) { + raise_from(PyExc_TypeError, msg.c_str()); + return nullptr; + } PyErr_SetString(PyExc_TypeError, msg.c_str()); return nullptr; - } else { - if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { - auto *pi = reinterpret_cast<instance *>(parent.ptr()); - self_value_and_holder.type->init_instance(pi, nullptr); - } - return result.ptr(); } + if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { + auto *pi = reinterpret_cast<instance *>(parent.ptr()); + self_value_and_holder.type->init_instance(pi, nullptr); + } + return result.ptr(); } }; @@ -945,11 +1157,7 @@ public: /// Create a new top-level Python module with the given name and docstring PYBIND11_DEPRECATED("Use PYBIND11_MODULE or module_::create_extension_module instead") explicit module_(const char *name, const char *doc = nullptr) { -#if PY_MAJOR_VERSION >= 3 *this = create_extension_module(name, doc, new PyModuleDef()); -#else - *this = create_extension_module(name, doc, nullptr); -#endif } /** \rst @@ -958,11 +1166,15 @@ public: details on the ``Extra&& ... extra`` argument, see section :ref:`extras`. \endrst */ template <typename Func, typename... Extra> - module_ &def(const char *name_, Func &&f, const Extra& ... extra) { - cpp_function func(std::forward<Func>(f), name(name_), scope(*this), - sibling(getattr(*this, name_, none())), extra...); + module_ &def(const char *name_, Func &&f, const Extra &...extra) { + cpp_function func(std::forward<Func>(f), + name(name_), + scope(*this), + sibling(getattr(*this, name_, none())), + extra...); // NB: allow overwriting here because cpp_function sets up a chain with the intention of - // overwriting (and has already checked internally that it isn't overwriting non-functions). + // overwriting (and has already checked internally that it isn't overwriting + // non-functions). add_object(name_, func, true /* overwrite */); return *this; } @@ -978,11 +1190,19 @@ public: py::module_ m3 = m2.def_submodule("subsub", "A submodule of 'example.sub'"); \endrst */ module_ def_submodule(const char *name, const char *doc = nullptr) { - std::string full_name = std::string(PyModule_GetName(m_ptr)) - + std::string(".") + std::string(name); - auto result = reinterpret_borrow<module_>(PyImport_AddModule(full_name.c_str())); - if (doc && options::show_user_defined_docstrings()) + const char *this_name = PyModule_GetName(m_ptr); + if (this_name == nullptr) { + throw error_already_set(); + } + std::string full_name = std::string(this_name) + '.' + name; + handle submodule = PyImport_AddModule(full_name.c_str()); + if (!submodule) { + throw error_already_set(); + } + auto result = reinterpret_borrow<module_>(submodule); + if (doc && options::show_user_defined_docstrings()) { result.attr("__doc__") = pybind11::str(doc); + } attr(name) = result; return result; } @@ -990,16 +1210,18 @@ public: /// Import and return a module or throws `error_already_set`. static module_ import(const char *name) { PyObject *obj = PyImport_ImportModule(name); - if (!obj) + if (!obj) { throw error_already_set(); + } return reinterpret_steal<module_>(obj); } /// Reload the module or throws `error_already_set`. void reload() { PyObject *obj = PyImport_ReloadModule(ptr()); - if (!obj) + if (!obj) { throw error_already_set(); + } *this = reinterpret_steal<module_>(obj); } @@ -1007,56 +1229,49 @@ public: Adds an object to the module using the given name. Throws if an object with the given name already exists. - ``overwrite`` should almost always be false: attempting to overwrite objects that pybind11 has - established will, in most cases, break things. + ``overwrite`` should almost always be false: attempting to overwrite objects that pybind11 + has established will, in most cases, break things. \endrst */ PYBIND11_NOINLINE void add_object(const char *name, handle obj, bool overwrite = false) { - if (!overwrite && hasattr(*this, name)) - pybind11_fail("Error during initialization: multiple incompatible definitions with name \"" + - std::string(name) + "\""); + if (!overwrite && hasattr(*this, name)) { + pybind11_fail( + "Error during initialization: multiple incompatible definitions with name \"" + + std::string(name) + "\""); + } PyModule_AddObject(ptr(), name, obj.inc_ref().ptr() /* steals a reference */); } -#if PY_MAJOR_VERSION >= 3 - using module_def = PyModuleDef; -#else - struct module_def {}; -#endif + using module_def = PyModuleDef; // TODO: Can this be removed (it was needed only for Python 2)? /** \rst Create a new top-level module that can be used as the main module of a C extension. - For Python 3, ``def`` should point to a statically allocated module_def. - For Python 2, ``def`` can be a nullptr and is completely ignored. + ``def`` should point to a statically allocated module_def. \endrst */ static module_ create_extension_module(const char *name, const char *doc, module_def *def) { -#if PY_MAJOR_VERSION >= 3 // module_def is PyModuleDef - def = new (def) PyModuleDef { // Placement new (not an allocation). - /* m_base */ PyModuleDef_HEAD_INIT, - /* m_name */ name, - /* m_doc */ options::show_user_defined_docstrings() ? doc : nullptr, - /* m_size */ -1, - /* m_methods */ nullptr, - /* m_slots */ nullptr, - /* m_traverse */ nullptr, - /* m_clear */ nullptr, - /* m_free */ nullptr - }; - auto m = PyModule_Create(def); -#else - // Ignore module_def *def; only necessary for Python 3 - (void) def; - auto m = Py_InitModule3(name, nullptr, options::show_user_defined_docstrings() ? doc : nullptr); -#endif + // Placement new (not an allocation). + def = new (def) + PyModuleDef{/* m_base */ PyModuleDef_HEAD_INIT, + /* m_name */ name, + /* m_doc */ options::show_user_defined_docstrings() ? doc : nullptr, + /* m_size */ -1, + /* m_methods */ nullptr, + /* m_slots */ nullptr, + /* m_traverse */ nullptr, + /* m_clear */ nullptr, + /* m_free */ nullptr}; + auto *m = PyModule_Create(def); if (m == nullptr) { - if (PyErr_Occurred()) + if (PyErr_Occurred()) { throw error_already_set(); + } pybind11_fail("Internal error in module_::create_extension_module()"); } - // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when returned from PyInit_... - // For Python 2, reinterpret_borrow is correct. + // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when + // returned from PyInit_... + // For Python 2, reinterpret_borrow was correct. return reinterpret_borrow<module_>(m); } }; @@ -1074,6 +1289,13 @@ inline dict globals() { return reinterpret_borrow<dict>(p ? p : module_::import("__main__").attr("__dict__").ptr()); } +template <typename... Args, typename = detail::enable_if_t<args_are_all_keyword_or_ds<Args...>()>> +PYBIND11_DEPRECATED("make_simple_namespace should be replaced with " + "py::module_::import(\"types\").attr(\"SimpleNamespace\") ") +object make_simple_namespace(Args &&...args_) { + return module_::import("types").attr("SimpleNamespace")(std::forward<Args>(args_)...); +} + PYBIND11_NAMESPACE_BEGIN(detail) /// Generic support for creating new Python heap types class generic_type : public object { @@ -1081,13 +1303,17 @@ public: PYBIND11_OBJECT_DEFAULT(generic_type, object, PyType_Check) protected: void initialize(const type_record &rec) { - if (rec.scope && hasattr(rec.scope, "__dict__") && rec.scope.attr("__dict__").contains(rec.name)) - pybind11_fail("generic_type: cannot initialize type \"" + std::string(rec.name) + - "\": an object with that name is already defined"); + if (rec.scope && hasattr(rec.scope, "__dict__") + && rec.scope.attr("__dict__").contains(rec.name)) { + pybind11_fail("generic_type: cannot initialize type \"" + std::string(rec.name) + + "\": an object with that name is already defined"); + } - if (rec.module_local ? get_local_type_info(*rec.type) : get_global_type_info(*rec.type)) - pybind11_fail("generic_type: type \"" + std::string(rec.name) + - "\" is already registered!"); + if ((rec.module_local ? get_local_type_info(*rec.type) : get_global_type_info(*rec.type)) + != nullptr) { + pybind11_fail("generic_type: type \"" + std::string(rec.name) + + "\" is already registered!"); + } m_ptr = make_new_python_type(rec); @@ -1109,19 +1335,23 @@ protected: auto &internals = get_internals(); auto tindex = std::type_index(*rec.type); tinfo->direct_conversions = &internals.direct_conversions[tindex]; - if (rec.module_local) - registered_local_types_cpp()[tindex] = tinfo; - else + if (rec.module_local) { + get_local_internals().registered_types_cpp[tindex] = tinfo; + } else { internals.registered_types_cpp[tindex] = tinfo; - internals.registered_types_py[(PyTypeObject *) m_ptr] = { tinfo }; + } + internals.registered_types_py[(PyTypeObject *) m_ptr] = {tinfo}; if (rec.bases.size() > 1 || rec.multiple_inheritance) { mark_parents_nonsimple(tinfo->type); tinfo->simple_ancestors = false; - } - else if (rec.bases.size() == 1) { - auto parent_tinfo = get_type_info((PyTypeObject *) rec.bases[0].ptr()); - tinfo->simple_ancestors = parent_tinfo->simple_ancestors; + } else if (rec.bases.size() == 1) { + auto *parent_tinfo = get_type_info((PyTypeObject *) rec.bases[0].ptr()); + assert(parent_tinfo != nullptr); + bool parent_simple_ancestors = parent_tinfo->simple_ancestors; + tinfo->simple_ancestors = parent_simple_ancestors; + // The parent can no longer be a simple type if it has MI and has a child + parent_tinfo->simple_type = parent_tinfo->simple_type && parent_simple_ancestors; } if (rec.module_local) { @@ -1135,25 +1365,25 @@ protected: void mark_parents_nonsimple(PyTypeObject *value) { auto t = reinterpret_borrow<tuple>(value->tp_bases); for (handle h : t) { - auto tinfo2 = get_type_info((PyTypeObject *) h.ptr()); - if (tinfo2) + auto *tinfo2 = get_type_info((PyTypeObject *) h.ptr()); + if (tinfo2) { tinfo2->simple_type = false; + } mark_parents_nonsimple((PyTypeObject *) h.ptr()); } } - void install_buffer_funcs( - buffer_info *(*get_buffer)(PyObject *, void *), - void *get_buffer_data) { - auto *type = (PyHeapTypeObject*) m_ptr; - auto tinfo = detail::get_type_info(&type->ht_type); + void install_buffer_funcs(buffer_info *(*get_buffer)(PyObject *, void *), + void *get_buffer_data) { + auto *type = (PyHeapTypeObject *) m_ptr; + auto *tinfo = detail::get_type_info(&type->ht_type); - if (!type->ht_type.tp_as_buffer) - pybind11_fail( - "To be able to register buffer protocol support for the type '" + - get_fully_qualified_tp_name(tinfo->type) + - "' the associated class<>(..) invocation must " - "include the pybind11::buffer_protocol() annotation!"); + if (!type->ht_type.tp_as_buffer) { + pybind11_fail("To be able to register buffer protocol support for the type '" + + get_fully_qualified_tp_name(tinfo->type) + + "' the associated class<>(..) invocation must " + "include the pybind11::buffer_protocol() annotation!"); + } tinfo->get_buffer = get_buffer; tinfo->get_buffer_data = get_buffer_data; @@ -1161,60 +1391,79 @@ protected: // rec_func must be set for either fget or fset. void def_property_static_impl(const char *name, - handle fget, handle fset, + handle fget, + handle fset, detail::function_record *rec_func) { - const auto is_static = rec_func && !(rec_func->is_method && rec_func->scope); - const auto has_doc = rec_func && rec_func->doc && pybind11::options::show_user_defined_docstrings(); - auto property = handle((PyObject *) (is_static ? get_internals().static_property_type - : &PyProperty_Type)); + const auto is_static = (rec_func != nullptr) && !(rec_func->is_method && rec_func->scope); + const auto has_doc = (rec_func != nullptr) && (rec_func->doc != nullptr) + && pybind11::options::show_user_defined_docstrings(); + auto property = handle( + (PyObject *) (is_static ? get_internals().static_property_type : &PyProperty_Type)); attr(name) = property(fget.ptr() ? fget : none(), fset.ptr() ? fset : none(), - /*deleter*/none(), + /*deleter*/ none(), pybind11::str(has_doc ? rec_func->doc : "")); } }; /// Set the pointer to operator new if it exists. The cast is needed because it can be overloaded. -template <typename T, typename = void_t<decltype(static_cast<void *(*)(size_t)>(T::operator new))>> -void set_operator_new(type_record *r) { r->operator_new = &T::operator new; } - -template <typename> void set_operator_new(...) { } +template <typename T, + typename = void_t<decltype(static_cast<void *(*) (size_t)>(T::operator new))>> +void set_operator_new(type_record *r) { + r->operator_new = &T::operator new; +} -template <typename T, typename SFINAE = void> struct has_operator_delete : std::false_type { }; -template <typename T> struct has_operator_delete<T, void_t<decltype(static_cast<void (*)(void *)>(T::operator delete))>> - : std::true_type { }; -template <typename T, typename SFINAE = void> struct has_operator_delete_size : std::false_type { }; -template <typename T> struct has_operator_delete_size<T, void_t<decltype(static_cast<void (*)(void *, size_t)>(T::operator delete))>> - : std::true_type { }; +template <typename> +void set_operator_new(...) {} + +template <typename T, typename SFINAE = void> +struct has_operator_delete : std::false_type {}; +template <typename T> +struct has_operator_delete<T, void_t<decltype(static_cast<void (*)(void *)>(T::operator delete))>> + : std::true_type {}; +template <typename T, typename SFINAE = void> +struct has_operator_delete_size : std::false_type {}; +template <typename T> +struct has_operator_delete_size< + T, + void_t<decltype(static_cast<void (*)(void *, size_t)>(T::operator delete))>> : std::true_type { +}; /// Call class-specific delete if it exists or global otherwise. Can also be an overload set. template <typename T, enable_if_t<has_operator_delete<T>::value, int> = 0> -void call_operator_delete(T *p, size_t, size_t) { T::operator delete(p); } -template <typename T, enable_if_t<!has_operator_delete<T>::value && has_operator_delete_size<T>::value, int> = 0> -void call_operator_delete(T *p, size_t s, size_t) { T::operator delete(p, s); } +void call_operator_delete(T *p, size_t, size_t) { + T::operator delete(p); +} +template <typename T, + enable_if_t<!has_operator_delete<T>::value && has_operator_delete_size<T>::value, int> + = 0> +void call_operator_delete(T *p, size_t s, size_t) { + T::operator delete(p, s); +} inline void call_operator_delete(void *p, size_t s, size_t a) { - (void)s; (void)a; - #if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) - if (a > __STDCPP_DEFAULT_NEW_ALIGNMENT__) { - #ifdef __cpp_sized_deallocation - ::operator delete(p, s, std::align_val_t(a)); - #else - ::operator delete(p, std::align_val_t(a)); - #endif - return; - } - #endif - #ifdef __cpp_sized_deallocation - ::operator delete(p, s); - #else - ::operator delete(p); - #endif + (void) s; + (void) a; +#if defined(__cpp_aligned_new) && (!defined(_MSC_VER) || _MSC_VER >= 1912) + if (a > __STDCPP_DEFAULT_NEW_ALIGNMENT__) { +# ifdef __cpp_sized_deallocation + ::operator delete(p, s, std::align_val_t(a)); +# else + ::operator delete(p, std::align_val_t(a)); +# endif + return; + } +#endif +#ifdef __cpp_sized_deallocation + ::operator delete(p, s); +#else + ::operator delete(p); +#endif } -inline void add_class_method(object& cls, const char *name_, const cpp_function &cf) { +inline void add_class_method(object &cls, const char *name_, const cpp_function &cf) { cls.attr(cf.name()) = cf; - if (strcmp(name_, "__eq__") == 0 && !cls.attr("__dict__").contains("__hash__")) { - cls.attr("__hash__") = none(); + if (std::strcmp(name_, "__eq__") == 0 && !cls.attr("__dict__").contains("__hash__")) { + cls.attr("__hash__") = none(); } } @@ -1223,30 +1472,37 @@ PYBIND11_NAMESPACE_END(detail) /// Given a pointer to a member function, cast it to its `Derived` version. /// Forward everything else unchanged. template <typename /*Derived*/, typename F> -auto method_adaptor(F &&f) -> decltype(std::forward<F>(f)) { return std::forward<F>(f); } +auto method_adaptor(F &&f) -> decltype(std::forward<F>(f)) { + return std::forward<F>(f); +} template <typename Derived, typename Return, typename Class, typename... Args> auto method_adaptor(Return (Class::*pmf)(Args...)) -> Return (Derived::*)(Args...) { - static_assert(detail::is_accessible_base_of<Class, Derived>::value, + static_assert( + detail::is_accessible_base_of<Class, Derived>::value, "Cannot bind an inaccessible base class method; use a lambda definition instead"); return pmf; } template <typename Derived, typename Return, typename Class, typename... Args> auto method_adaptor(Return (Class::*pmf)(Args...) const) -> Return (Derived::*)(Args...) const { - static_assert(detail::is_accessible_base_of<Class, Derived>::value, + static_assert( + detail::is_accessible_base_of<Class, Derived>::value, "Cannot bind an inaccessible base class method; use a lambda definition instead"); return pmf; } template <typename type_, typename... options> class class_ : public detail::generic_type { - template <typename T> using is_holder = detail::is_holder_type<type_, T>; - template <typename T> using is_subtype = detail::is_strict_base_of<type_, T>; - template <typename T> using is_base = detail::is_strict_base_of<T, type_>; + template <typename T> + using is_holder = detail::is_holder_type<type_, T>; + template <typename T> + using is_subtype = detail::is_strict_base_of<type_, T>; + template <typename T> + using is_base = detail::is_strict_base_of<T, type_>; // struct instead of using here to help MSVC: - template <typename T> struct is_valid_class_option : - detail::any_of<is_holder<T>, is_subtype<T>, is_base<T>> {}; + template <typename T> + struct is_valid_class_option : detail::any_of<is_holder<T>, is_subtype<T>, is_base<T>> {}; public: using type = type_; @@ -1255,23 +1511,24 @@ public: using holder_type = detail::exactly_one_t<is_holder, std::unique_ptr<type>, options...>; static_assert(detail::all_of<is_valid_class_option<options>...>::value, - "Unknown/invalid class_ template parameters provided"); + "Unknown/invalid class_ template parameters provided"); static_assert(!has_alias || std::is_polymorphic<type>::value, - "Cannot use an alias class with a non-polymorphic type"); + "Cannot use an alias class with a non-polymorphic type"); PYBIND11_OBJECT(class_, generic_type, PyType_Check) template <typename... Extra> - class_(handle scope, const char *name, const Extra &... extra) { + class_(handle scope, const char *name, const Extra &...extra) { using namespace detail; // MI can only be specified via class_ template options, not constructor parameters static_assert( none_of<is_pyobject<Extra>...>::value || // no base class arguments, or: - ( constexpr_sum(is_pyobject<Extra>::value...) == 1 && // Exactly one base - constexpr_sum(is_base<options>::value...) == 0 && // no template option bases - none_of<std::is_same<multiple_inheritance, Extra>...>::value), // no multiple_inheritance attr + (constexpr_sum(is_pyobject<Extra>::value...) == 1 && // Exactly one base + constexpr_sum(is_base<options>::value...) == 0 && // no template option bases + // no multiple_inheritance attr + none_of<std::is_same<multiple_inheritance, Extra>...>::value), "Error: multiple inheritance bases must be specified via class_ template options"); type_record record; @@ -1279,7 +1536,7 @@ public: record.name = name; record.type = &typeid(type); record.type_size = sizeof(conditional_t<has_alias, type_alias, type>); - record.type_align = alignof(conditional_t<has_alias, type_alias, type>&); + record.type_align = alignof(conditional_t<has_alias, type_alias, type> &); record.holder_size = sizeof(holder_type); record.init_instance = init_instance; record.dealloc = dealloc; @@ -1296,8 +1553,10 @@ public: generic_type::initialize(record); if (has_alias) { - auto &instances = record.module_local ? registered_local_types_cpp() : get_internals().registered_types_cpp; - instances[std::type_index(typeid(type_alias))] = instances[std::type_index(typeid(type))]; + auto &instances = record.module_local ? get_local_internals().registered_types_cpp + : get_internals().registered_types_cpp; + instances[std::type_index(typeid(type_alias))] + = instances[std::type_index(typeid(type))]; } } @@ -1309,52 +1568,61 @@ public: } template <typename Base, detail::enable_if_t<!is_base<Base>::value, int> = 0> - static void add_base(detail::type_record &) { } + static void add_base(detail::type_record &) {} template <typename Func, typename... Extra> - class_ &def(const char *name_, Func&& f, const Extra&... extra) { - cpp_function cf(method_adaptor<type>(std::forward<Func>(f)), name(name_), is_method(*this), - sibling(getattr(*this, name_, none())), extra...); + class_ &def(const char *name_, Func &&f, const Extra &...extra) { + cpp_function cf(method_adaptor<type>(std::forward<Func>(f)), + name(name_), + is_method(*this), + sibling(getattr(*this, name_, none())), + extra...); add_class_method(*this, name_, cf); return *this; } - template <typename Func, typename... Extra> class_ & - def_static(const char *name_, Func &&f, const Extra&... extra) { + template <typename Func, typename... Extra> + class_ &def_static(const char *name_, Func &&f, const Extra &...extra) { static_assert(!std::is_member_function_pointer<Func>::value, - "def_static(...) called with a non-static member function pointer"); - cpp_function cf(std::forward<Func>(f), name(name_), scope(*this), - sibling(getattr(*this, name_, none())), extra...); - attr(cf.name()) = staticmethod(cf); + "def_static(...) called with a non-static member function pointer"); + cpp_function cf(std::forward<Func>(f), + name(name_), + scope(*this), + sibling(getattr(*this, name_, none())), + extra...); + auto cf_name = cf.name(); + attr(std::move(cf_name)) = staticmethod(std::move(cf)); return *this; } - template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra> - class_ &def(const detail::op_<id, ot, L, R> &op, const Extra&... extra) { + template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0> + class_ &def(const T &op, const Extra &...extra) { op.execute(*this, extra...); return *this; } - template <detail::op_id id, detail::op_type ot, typename L, typename R, typename... Extra> - class_ & def_cast(const detail::op_<id, ot, L, R> &op, const Extra&... extra) { + template <typename T, typename... Extra, detail::enable_if_t<T::op_enable_if_hook, int> = 0> + class_ &def_cast(const T &op, const Extra &...extra) { op.execute_cast(*this, extra...); return *this; } template <typename... Args, typename... Extra> - class_ &def(const detail::initimpl::constructor<Args...> &init, const Extra&... extra) { + class_ &def(const detail::initimpl::constructor<Args...> &init, const Extra &...extra) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(init); init.execute(*this, extra...); return *this; } template <typename... Args, typename... Extra> - class_ &def(const detail::initimpl::alias_constructor<Args...> &init, const Extra&... extra) { + class_ &def(const detail::initimpl::alias_constructor<Args...> &init, const Extra &...extra) { + PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(init); init.execute(*this, extra...); return *this; } template <typename... Args, typename... Extra> - class_ &def(detail::initimpl::factory<Args...> &&init, const Extra&... extra) { + class_ &def(detail::initimpl::factory<Args...> &&init, const Extra &...extra) { std::move(init).execute(*this, extra...); return *this; } @@ -1366,135 +1634,171 @@ public: } template <typename Func> - class_& def_buffer(Func &&func) { - struct capture { Func func; }; - auto *ptr = new capture { std::forward<Func>(func) }; - install_buffer_funcs([](PyObject *obj, void *ptr) -> buffer_info* { - detail::make_caster<type> caster; - if (!caster.load(obj, false)) - return nullptr; - return new buffer_info(((capture *) ptr)->func(caster)); - }, ptr); + class_ &def_buffer(Func &&func) { + struct capture { + Func func; + }; + auto *ptr = new capture{std::forward<Func>(func)}; + install_buffer_funcs( + [](PyObject *obj, void *ptr) -> buffer_info * { + detail::make_caster<type> caster; + if (!caster.load(obj, false)) { + return nullptr; + } + return new buffer_info(((capture *) ptr)->func(std::move(caster))); + }, + ptr); weakref(m_ptr, cpp_function([ptr](handle wr) { - delete ptr; - wr.dec_ref(); - })).release(); + delete ptr; + wr.dec_ref(); + })) + .release(); return *this; } template <typename Return, typename Class, typename... Args> class_ &def_buffer(Return (Class::*func)(Args...)) { - return def_buffer([func] (type &obj) { return (obj.*func)(); }); + return def_buffer([func](type &obj) { return (obj.*func)(); }); } template <typename Return, typename Class, typename... Args> class_ &def_buffer(Return (Class::*func)(Args...) const) { - return def_buffer([func] (const type &obj) { return (obj.*func)(); }); + return def_buffer([func](const type &obj) { return (obj.*func)(); }); } template <typename C, typename D, typename... Extra> - class_ &def_readwrite(const char *name, D C::*pm, const Extra&... extra) { - static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value, "def_readwrite() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D &{ return c.*pm; }, is_method(*this)), - fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this)); + class_ &def_readwrite(const char *name, D C::*pm, const Extra &...extra) { + static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value, + "def_readwrite() requires a class member (or base class member)"); + cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)), + fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this)); def_property(name, fget, fset, return_value_policy::reference_internal, extra...); return *this; } template <typename C, typename D, typename... Extra> - class_ &def_readonly(const char *name, const D C::*pm, const Extra& ...extra) { - static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value, "def_readonly() requires a class member (or base class member)"); - cpp_function fget([pm](const type &c) -> const D &{ return c.*pm; }, is_method(*this)); + class_ &def_readonly(const char *name, const D C::*pm, const Extra &...extra) { + static_assert(std::is_same<C, type>::value || std::is_base_of<C, type>::value, + "def_readonly() requires a class member (or base class member)"); + cpp_function fget([pm](const type &c) -> const D & { return c.*pm; }, is_method(*this)); def_property_readonly(name, fget, return_value_policy::reference_internal, extra...); return *this; } template <typename D, typename... Extra> - class_ &def_readwrite_static(const char *name, D *pm, const Extra& ...extra) { - cpp_function fget([pm](object) -> const D &{ return *pm; }, scope(*this)), - fset([pm](object, const D &value) { *pm = value; }, scope(*this)); + class_ &def_readwrite_static(const char *name, D *pm, const Extra &...extra) { + cpp_function fget([pm](const object &) -> const D & { return *pm; }, scope(*this)), + fset([pm](const object &, const D &value) { *pm = value; }, scope(*this)); def_property_static(name, fget, fset, return_value_policy::reference, extra...); return *this; } template <typename D, typename... Extra> - class_ &def_readonly_static(const char *name, const D *pm, const Extra& ...extra) { - cpp_function fget([pm](object) -> const D &{ return *pm; }, scope(*this)); + class_ &def_readonly_static(const char *name, const D *pm, const Extra &...extra) { + cpp_function fget([pm](const object &) -> const D & { return *pm; }, scope(*this)); def_property_readonly_static(name, fget, return_value_policy::reference, extra...); return *this; } /// Uses return_value_policy::reference_internal by default template <typename Getter, typename... Extra> - class_ &def_property_readonly(const char *name, const Getter &fget, const Extra& ...extra) { - return def_property_readonly(name, cpp_function(method_adaptor<type>(fget)), - return_value_policy::reference_internal, extra...); + class_ &def_property_readonly(const char *name, const Getter &fget, const Extra &...extra) { + return def_property_readonly(name, + cpp_function(method_adaptor<type>(fget)), + return_value_policy::reference_internal, + extra...); } /// Uses cpp_function's return_value_policy by default template <typename... Extra> - class_ &def_property_readonly(const char *name, const cpp_function &fget, const Extra& ...extra) { + class_ & + def_property_readonly(const char *name, const cpp_function &fget, const Extra &...extra) { return def_property(name, fget, nullptr, extra...); } /// Uses return_value_policy::reference by default template <typename Getter, typename... Extra> - class_ &def_property_readonly_static(const char *name, const Getter &fget, const Extra& ...extra) { - return def_property_readonly_static(name, cpp_function(fget), return_value_policy::reference, extra...); + class_ & + def_property_readonly_static(const char *name, const Getter &fget, const Extra &...extra) { + return def_property_readonly_static( + name, cpp_function(fget), return_value_policy::reference, extra...); } /// Uses cpp_function's return_value_policy by default template <typename... Extra> - class_ &def_property_readonly_static(const char *name, const cpp_function &fget, const Extra& ...extra) { + class_ &def_property_readonly_static(const char *name, + const cpp_function &fget, + const Extra &...extra) { return def_property_static(name, fget, nullptr, extra...); } /// Uses return_value_policy::reference_internal by default template <typename Getter, typename Setter, typename... Extra> - class_ &def_property(const char *name, const Getter &fget, const Setter &fset, const Extra& ...extra) { - return def_property(name, fget, cpp_function(method_adaptor<type>(fset)), extra...); + class_ & + def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { + return def_property( + name, fget, cpp_function(method_adaptor<type>(fset), is_setter()), extra...); } template <typename Getter, typename... Extra> - class_ &def_property(const char *name, const Getter &fget, const cpp_function &fset, const Extra& ...extra) { - return def_property(name, cpp_function(method_adaptor<type>(fget)), fset, - return_value_policy::reference_internal, extra...); + class_ &def_property(const char *name, + const Getter &fget, + const cpp_function &fset, + const Extra &...extra) { + return def_property(name, + cpp_function(method_adaptor<type>(fget)), + fset, + return_value_policy::reference_internal, + extra...); } /// Uses cpp_function's return_value_policy by default template <typename... Extra> - class_ &def_property(const char *name, const cpp_function &fget, const cpp_function &fset, const Extra& ...extra) { + class_ &def_property(const char *name, + const cpp_function &fget, + const cpp_function &fset, + const Extra &...extra) { return def_property_static(name, fget, fset, is_method(*this), extra...); } /// Uses return_value_policy::reference by default template <typename Getter, typename... Extra> - class_ &def_property_static(const char *name, const Getter &fget, const cpp_function &fset, const Extra& ...extra) { - return def_property_static(name, cpp_function(fget), fset, return_value_policy::reference, extra...); + class_ &def_property_static(const char *name, + const Getter &fget, + const cpp_function &fset, + const Extra &...extra) { + return def_property_static( + name, cpp_function(fget), fset, return_value_policy::reference, extra...); } /// Uses cpp_function's return_value_policy by default template <typename... Extra> - class_ &def_property_static(const char *name, const cpp_function &fget, const cpp_function &fset, const Extra& ...extra) { - static_assert( 0 == detail::constexpr_sum(std::is_base_of<arg, Extra>::value...), + class_ &def_property_static(const char *name, + const cpp_function &fget, + const cpp_function &fset, + const Extra &...extra) { + static_assert(0 == detail::constexpr_sum(std::is_base_of<arg, Extra>::value...), "Argument annotations are not allowed for properties"); auto rec_fget = get_function_record(fget), rec_fset = get_function_record(fset); auto *rec_active = rec_fget; if (rec_fget) { - char *doc_prev = rec_fget->doc; /* 'extra' field may include a property-specific documentation string */ - detail::process_attributes<Extra...>::init(extra..., rec_fget); - if (rec_fget->doc && rec_fget->doc != doc_prev) { - free(doc_prev); - rec_fget->doc = strdup(rec_fget->doc); - } + char *doc_prev = rec_fget->doc; /* 'extra' field may include a property-specific + documentation string */ + detail::process_attributes<Extra...>::init(extra..., rec_fget); + if (rec_fget->doc && rec_fget->doc != doc_prev) { + std::free(doc_prev); + rec_fget->doc = PYBIND11_COMPAT_STRDUP(rec_fget->doc); + } } if (rec_fset) { char *doc_prev = rec_fset->doc; detail::process_attributes<Extra...>::init(extra..., rec_fset); if (rec_fset->doc && rec_fset->doc != doc_prev) { - free(doc_prev); - rec_fset->doc = strdup(rec_fset->doc); + std::free(doc_prev); + rec_fset->doc = PYBIND11_COMPAT_STRDUP(rec_fset->doc); + } + if (!rec_active) { + rec_active = rec_fset; } - if (! rec_active) rec_active = rec_fset; } def_property_static_impl(name, fget, fset, rec_active); return *this; @@ -1503,16 +1807,17 @@ public: private: /// Initialize holder object, variant 1: object derives from enable_shared_from_this template <typename T> - static void init_holder(detail::instance *inst, detail::value_and_holder &v_h, - const holder_type * /* unused */, const std::enable_shared_from_this<T> * /* dummy */) { - try { - auto sh = std::dynamic_pointer_cast<typename holder_type::element_type>( - v_h.value_ptr<type>()->shared_from_this()); - if (sh) { - new (std::addressof(v_h.holder<holder_type>())) holder_type(std::move(sh)); - v_h.set_holder_constructed(); - } - } catch (const std::bad_weak_ptr &) {} + static void init_holder(detail::instance *inst, + detail::value_and_holder &v_h, + const holder_type * /* unused */, + const std::enable_shared_from_this<T> * /* dummy */) { + + auto sh = std::dynamic_pointer_cast<typename holder_type::element_type>( + detail::try_get_shared_from_this(v_h.value_ptr<type>())); + if (sh) { + new (std::addressof(v_h.holder<holder_type>())) holder_type(std::move(sh)); + v_h.set_holder_constructed(); + } if (!v_h.holder_constructed() && inst->owned) { new (std::addressof(v_h.holder<holder_type>())) holder_type(v_h.value_ptr<type>()); @@ -1521,30 +1826,37 @@ private: } static void init_holder_from_existing(const detail::value_and_holder &v_h, - const holder_type *holder_ptr, std::true_type /*is_copy_constructible*/) { - new (std::addressof(v_h.holder<holder_type>())) holder_type(*reinterpret_cast<const holder_type *>(holder_ptr)); + const holder_type *holder_ptr, + std::true_type /*is_copy_constructible*/) { + new (std::addressof(v_h.holder<holder_type>())) + holder_type(*reinterpret_cast<const holder_type *>(holder_ptr)); } static void init_holder_from_existing(const detail::value_and_holder &v_h, - const holder_type *holder_ptr, std::false_type /*is_copy_constructible*/) { - new (std::addressof(v_h.holder<holder_type>())) holder_type(std::move(*const_cast<holder_type *>(holder_ptr))); - } - - /// Initialize holder object, variant 2: try to construct from existing holder object, if possible - static void init_holder(detail::instance *inst, detail::value_and_holder &v_h, - const holder_type *holder_ptr, const void * /* dummy -- not enable_shared_from_this<T>) */) { + const holder_type *holder_ptr, + std::false_type /*is_copy_constructible*/) { + new (std::addressof(v_h.holder<holder_type>())) + holder_type(std::move(*const_cast<holder_type *>(holder_ptr))); + } + + /// Initialize holder object, variant 2: try to construct from existing holder object, if + /// possible + static void init_holder(detail::instance *inst, + detail::value_and_holder &v_h, + const holder_type *holder_ptr, + const void * /* dummy -- not enable_shared_from_this<T>) */) { if (holder_ptr) { init_holder_from_existing(v_h, holder_ptr, std::is_copy_constructible<holder_type>()); v_h.set_holder_constructed(); - } else if (inst->owned || detail::always_construct_holder<holder_type>::value) { + } else if (detail::always_construct_holder<holder_type>::value || inst->owned) { new (std::addressof(v_h.holder<holder_type>())) holder_type(v_h.value_ptr<type>()); v_h.set_holder_constructed(); } } /// Performs instance initialization including constructing a holder and registering the known - /// instance. Should be called as soon as the `type` value_ptr is set for an instance. Takes an - /// optional pointer to an existing holder to use; if not specified and the instance is + /// instance. Should be called as soon as the `type` value_ptr is set for an instance. Takes + /// an optional pointer to an existing holder to use; if not specified and the instance is /// `.owned`, a new holder will be constructed to manage the value pointer. static void init_instance(detail::instance *inst, const void *holder_ptr) { auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type))); @@ -1567,35 +1879,55 @@ private: if (v_h.holder_constructed()) { v_h.holder<holder_type>().~holder_type(); v_h.set_holder_constructed(false); - } - else { - detail::call_operator_delete(v_h.value_ptr<type>(), - v_h.type->type_size, - v_h.type->type_align - ); + } else { + detail::call_operator_delete( + v_h.value_ptr<type>(), v_h.type->type_size, v_h.type->type_align); } v_h.value_ptr() = nullptr; } static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow<capsule>(PyCFunction_GET_SELF(h.ptr())) - : nullptr; + if (!h) { + return nullptr; + } + + handle func_self = PyCFunction_GET_SELF(h.ptr()); + if (!func_self) { + throw error_already_set(); + } + if (!isinstance<capsule>(func_self)) { + return nullptr; + } + auto cap = reinterpret_borrow<capsule>(func_self); + if (!detail::is_function_record_capsule(cap)) { + return nullptr; + } + return cap.get_pointer<detail::function_record>(); } }; /// Binds an existing constructor taking arguments Args... -template <typename... Args> detail::initimpl::constructor<Args...> init() { return {}; } +template <typename... Args> +detail::initimpl::constructor<Args...> init() { + return {}; +} /// Like `init<Args...>()`, but the instance is always constructed through the alias class (even /// when not inheriting on the Python side). -template <typename... Args> detail::initimpl::alias_constructor<Args...> init_alias() { return {}; } +template <typename... Args> +detail::initimpl::alias_constructor<Args...> init_alias() { + return {}; +} /// Binds a factory function as a constructor template <typename Func, typename Ret = detail::initimpl::factory<Func>> -Ret init(Func &&f) { return {std::forward<Func>(f)}; } +Ret init(Func &&f) { + return {std::forward<Func>(f)}; +} -/// Dual-argument factory function: the first function is called when no alias is needed, the second -/// when an alias is needed (i.e. due to python-side inheritance). Arguments must be identical. +/// Dual-argument factory function: the first function is called when no alias is needed, the +/// second when an alias is needed (i.e. due to python-side inheritance). Arguments must be +/// identical. template <typename CFunc, typename AFunc, typename Ret = detail::initimpl::factory<CFunc, AFunc>> Ret init(CFunc &&c, AFunc &&a) { return {std::forward<CFunc>(c), std::forward<AFunc>(a)}; @@ -1613,14 +1945,15 @@ PYBIND11_NAMESPACE_BEGIN(detail) inline str enum_name(handle arg) { dict entries = arg.get_type().attr("__entries"); for (auto kv : entries) { - if (handle(kv.second[int_(0)]).equal(arg)) + if (handle(kv.second[int_(0)]).equal(arg)) { return pybind11::str(kv.first); + } } return "???"; } struct enum_base { - enum_base(handle base, handle parent) : m_base(base), m_parent(parent) { } + enum_base(const handle &base, const handle &parent) : m_base(base), m_parent(parent) {} PYBIND11_NOINLINE void init(bool is_arithmetic, bool is_convertible) { m_base.attr("__entries") = dict(); @@ -1628,184 +1961,256 @@ struct enum_base { auto static_property = handle((PyObject *) get_internals().static_property_type); m_base.attr("__repr__") = cpp_function( - [](object arg) -> str { + [](const object &arg) -> str { handle type = type::handle_of(arg); object type_name = type.attr("__name__"); - return pybind11::str("<{}.{}: {}>").format(type_name, enum_name(arg), int_(arg)); - }, name("__repr__"), is_method(m_base) - ); + return pybind11::str("<{}.{}: {}>") + .format(std::move(type_name), enum_name(arg), int_(arg)); + }, + name("__repr__"), + is_method(m_base)); m_base.attr("name") = property(cpp_function(&enum_name, name("name"), is_method(m_base))); m_base.attr("__str__") = cpp_function( [](handle arg) -> str { object type_name = type::handle_of(arg).attr("__name__"); - return pybind11::str("{}.{}").format(type_name, enum_name(arg)); - }, name("name"), is_method(m_base) - ); - - m_base.attr("__doc__") = static_property(cpp_function( - [](handle arg) -> std::string { - std::string docstring; - dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) - docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; - docstring += "Members:"; - for (auto kv : entries) { - auto key = std::string(pybind11::str(kv.first)); - auto comment = kv.second[int_(1)]; - docstring += "\n\n " + key; - if (!comment.is_none()) - docstring += " : " + (std::string) pybind11::str(comment); - } - return docstring; - }, name("__doc__") - ), none(), none(), ""); + return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg)); + }, + name("name"), + is_method(m_base)); + + if (options::show_enum_members_docstring()) { + m_base.attr("__doc__") = static_property( + cpp_function( + [](handle arg) -> std::string { + std::string docstring; + dict entries = arg.attr("__entries"); + if (((PyTypeObject *) arg.ptr())->tp_doc) { + docstring += std::string( + reinterpret_cast<PyTypeObject *>(arg.ptr())->tp_doc); + docstring += "\n\n"; + } + docstring += "Members:"; + for (auto kv : entries) { + auto key = std::string(pybind11::str(kv.first)); + auto comment = kv.second[int_(1)]; + docstring += "\n\n "; + docstring += key; + if (!comment.is_none()) { + docstring += " : "; + docstring += pybind11::str(comment).cast<std::string>(); + } + } + return docstring; + }, + name("__doc__")), + none(), + none(), + ""); + } m_base.attr("__members__") = static_property(cpp_function( - [](handle arg) -> dict { - dict entries = arg.attr("__entries"), m; - for (auto kv : entries) - m[kv.first] = kv.second[int_(0)]; - return m; - }, name("__members__")), none(), none(), "" - ); - - #define PYBIND11_ENUM_OP_STRICT(op, expr, strict_behavior) \ - m_base.attr(op) = cpp_function( \ - [](object a, object b) { \ - if (!type::handle_of(a).is(type::handle_of(b))) \ - strict_behavior; \ - return expr; \ - }, \ - name(op), is_method(m_base), arg("other")) - - #define PYBIND11_ENUM_OP_CONV(op, expr) \ - m_base.attr(op) = cpp_function( \ - [](object a_, object b_) { \ - int_ a(a_), b(b_); \ - return expr; \ - }, \ - name(op), is_method(m_base), arg("other")) - - #define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \ - m_base.attr(op) = cpp_function( \ - [](object a_, object b) { \ - int_ a(a_); \ - return expr; \ - }, \ - name(op), is_method(m_base), arg("other")) + [](handle arg) -> dict { + dict entries = arg.attr("__entries"), + m; + for (auto kv : entries) { + m[kv.first] = kv.second[int_(0)]; + } + return m; + }, + name("__members__")), + none(), + none(), + ""); + +#define PYBIND11_ENUM_OP_STRICT(op, expr, strict_behavior) \ + m_base.attr(op) = cpp_function( \ + [](const object &a, const object &b) { \ + if (!type::handle_of(a).is(type::handle_of(b))) \ + strict_behavior; /* NOLINT(bugprone-macro-parentheses) */ \ + return expr; \ + }, \ + name(op), \ + is_method(m_base), \ + arg("other")) + +#define PYBIND11_ENUM_OP_CONV(op, expr) \ + m_base.attr(op) = cpp_function( \ + [](const object &a_, const object &b_) { \ + int_ a(a_), b(b_); \ + return expr; \ + }, \ + name(op), \ + is_method(m_base), \ + arg("other")) + +#define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \ + m_base.attr(op) = cpp_function( \ + [](const object &a_, const object &b) { \ + int_ a(a_); \ + return expr; \ + }, \ + name(op), \ + is_method(m_base), \ + arg("other")) if (is_convertible) { - PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b)); - PYBIND11_ENUM_OP_CONV_LHS("__ne__", b.is_none() || !a.equal(b)); + PYBIND11_ENUM_OP_CONV_LHS("__eq__", !b.is_none() && a.equal(b)); + PYBIND11_ENUM_OP_CONV_LHS("__ne__", b.is_none() || !a.equal(b)); if (is_arithmetic) { - PYBIND11_ENUM_OP_CONV("__lt__", a < b); - PYBIND11_ENUM_OP_CONV("__gt__", a > b); - PYBIND11_ENUM_OP_CONV("__le__", a <= b); - PYBIND11_ENUM_OP_CONV("__ge__", a >= b); - PYBIND11_ENUM_OP_CONV("__and__", a & b); - PYBIND11_ENUM_OP_CONV("__rand__", a & b); - PYBIND11_ENUM_OP_CONV("__or__", a | b); - PYBIND11_ENUM_OP_CONV("__ror__", a | b); - PYBIND11_ENUM_OP_CONV("__xor__", a ^ b); - PYBIND11_ENUM_OP_CONV("__rxor__", a ^ b); - m_base.attr("__invert__") = cpp_function( - [](object arg) { return ~(int_(arg)); }, name("__invert__"), is_method(m_base)); + PYBIND11_ENUM_OP_CONV("__lt__", a < b); + PYBIND11_ENUM_OP_CONV("__gt__", a > b); + PYBIND11_ENUM_OP_CONV("__le__", a <= b); + PYBIND11_ENUM_OP_CONV("__ge__", a >= b); + PYBIND11_ENUM_OP_CONV("__and__", a & b); + PYBIND11_ENUM_OP_CONV("__rand__", a & b); + PYBIND11_ENUM_OP_CONV("__or__", a | b); + PYBIND11_ENUM_OP_CONV("__ror__", a | b); + PYBIND11_ENUM_OP_CONV("__xor__", a ^ b); + PYBIND11_ENUM_OP_CONV("__rxor__", a ^ b); + m_base.attr("__invert__") + = cpp_function([](const object &arg) { return ~(int_(arg)); }, + name("__invert__"), + is_method(m_base)); } } else { - PYBIND11_ENUM_OP_STRICT("__eq__", int_(a).equal(int_(b)), return false); + PYBIND11_ENUM_OP_STRICT("__eq__", int_(a).equal(int_(b)), return false); PYBIND11_ENUM_OP_STRICT("__ne__", !int_(a).equal(int_(b)), return true); if (is_arithmetic) { - #define PYBIND11_THROW throw type_error("Expected an enumeration of matching type!"); - PYBIND11_ENUM_OP_STRICT("__lt__", int_(a) < int_(b), PYBIND11_THROW); - PYBIND11_ENUM_OP_STRICT("__gt__", int_(a) > int_(b), PYBIND11_THROW); +#define PYBIND11_THROW throw type_error("Expected an enumeration of matching type!"); + PYBIND11_ENUM_OP_STRICT("__lt__", int_(a) < int_(b), PYBIND11_THROW); + PYBIND11_ENUM_OP_STRICT("__gt__", int_(a) > int_(b), PYBIND11_THROW); PYBIND11_ENUM_OP_STRICT("__le__", int_(a) <= int_(b), PYBIND11_THROW); PYBIND11_ENUM_OP_STRICT("__ge__", int_(a) >= int_(b), PYBIND11_THROW); - #undef PYBIND11_THROW +#undef PYBIND11_THROW } } - #undef PYBIND11_ENUM_OP_CONV_LHS - #undef PYBIND11_ENUM_OP_CONV - #undef PYBIND11_ENUM_OP_STRICT +#undef PYBIND11_ENUM_OP_CONV_LHS +#undef PYBIND11_ENUM_OP_CONV +#undef PYBIND11_ENUM_OP_STRICT m_base.attr("__getstate__") = cpp_function( - [](object arg) { return int_(arg); }, name("__getstate__"), is_method(m_base)); + [](const object &arg) { return int_(arg); }, name("__getstate__"), is_method(m_base)); m_base.attr("__hash__") = cpp_function( - [](object arg) { return int_(arg); }, name("__hash__"), is_method(m_base)); + [](const object &arg) { return int_(arg); }, name("__hash__"), is_method(m_base)); } - PYBIND11_NOINLINE void value(char const* name_, object value, const char *doc = nullptr) { + PYBIND11_NOINLINE void value(char const *name_, object value, const char *doc = nullptr) { dict entries = m_base.attr("__entries"); str name(name_); if (entries.contains(name)) { std::string type_name = (std::string) str(m_base.attr("__name__")); - throw value_error(type_name + ": element \"" + std::string(name_) + "\" already exists!"); + throw value_error(std::move(type_name) + ": element \"" + std::string(name_) + + "\" already exists!"); } - entries[name] = std::make_pair(value, doc); - m_base.attr(name) = value; + entries[name] = pybind11::make_tuple(value, doc); + m_base.attr(std::move(name)) = std::move(value); } PYBIND11_NOINLINE void export_values() { dict entries = m_base.attr("__entries"); - for (auto kv : entries) + for (auto kv : entries) { m_parent.attr(kv.first) = kv.second[int_(0)]; + } } handle m_base; handle m_parent; }; +template <bool is_signed, size_t length> +struct equivalent_integer {}; +template <> +struct equivalent_integer<true, 1> { + using type = int8_t; +}; +template <> +struct equivalent_integer<false, 1> { + using type = uint8_t; +}; +template <> +struct equivalent_integer<true, 2> { + using type = int16_t; +}; +template <> +struct equivalent_integer<false, 2> { + using type = uint16_t; +}; +template <> +struct equivalent_integer<true, 4> { + using type = int32_t; +}; +template <> +struct equivalent_integer<false, 4> { + using type = uint32_t; +}; +template <> +struct equivalent_integer<true, 8> { + using type = int64_t; +}; +template <> +struct equivalent_integer<false, 8> { + using type = uint64_t; +}; + +template <typename IntLike> +using equivalent_integer_t = + typename equivalent_integer<std::is_signed<IntLike>::value, sizeof(IntLike)>::type; + PYBIND11_NAMESPACE_END(detail) /// Binds C++ enumerations and enumeration classes to Python -template <typename Type> class enum_ : public class_<Type> { +template <typename Type> +class enum_ : public class_<Type> { public: using Base = class_<Type>; - using Base::def; using Base::attr; + using Base::def; using Base::def_property_readonly; using Base::def_property_readonly_static; - using Scalar = typename std::underlying_type<Type>::type; + using Underlying = typename std::underlying_type<Type>::type; + // Scalar is the integer representation of underlying type + using Scalar = detail::conditional_t<detail::any_of<detail::is_std_char_type<Underlying>, + std::is_same<Underlying, bool>>::value, + detail::equivalent_integer_t<Underlying>, + Underlying>; template <typename... Extra> - enum_(const handle &scope, const char *name, const Extra&... extra) - : class_<Type>(scope, name, extra...), m_base(*this, scope) { + enum_(const handle &scope, const char *name, const Extra &...extra) + : class_<Type>(scope, name, extra...), m_base(*this, scope) { constexpr bool is_arithmetic = detail::any_of<std::is_same<arithmetic, Extra>...>::value; - constexpr bool is_convertible = std::is_convertible<Type, Scalar>::value; + constexpr bool is_convertible = std::is_convertible<Type, Underlying>::value; m_base.init(is_arithmetic, is_convertible); def(init([](Scalar i) { return static_cast<Type>(i); }), arg("value")); def_property_readonly("value", [](Type value) { return (Scalar) value; }); def("__int__", [](Type value) { return (Scalar) value; }); - #if PY_MAJOR_VERSION < 3 - def("__long__", [](Type value) { return (Scalar) value; }); - #endif - #if PY_MAJOR_VERSION > 3 || (PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION >= 8) - def("__index__", [](Type value) { return (Scalar) value; }); - #endif - + def("__index__", [](Type value) { return (Scalar) value; }); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { - detail::initimpl::setstate<Base>(v_h, static_cast<Type>(arg), - Py_TYPE(v_h.inst) != v_h.type->type); }, + detail::initimpl::setstate<Base>( + v_h, static_cast<Type>(arg), Py_TYPE(v_h.inst) != v_h.type->type); + }, detail::is_new_style_constructor(), - pybind11::name("__setstate__"), is_method(*this), arg("state")); + pybind11::name("__setstate__"), + is_method(*this), + arg("state")); } /// Export enumeration entries into the parent scope - enum_& export_values() { + enum_ &export_values() { m_base.export_values(); return *this; } /// Add an enumeration entry - enum_& value(char const* name, Type value, const char *doc = nullptr) { + enum_ &value(char const *name, Type value, const char *doc = nullptr) { m_base.value(name, pybind11::cast(value, return_value_policy::copy), doc); return *this; } @@ -1816,26 +2221,28 @@ private: PYBIND11_NAMESPACE_BEGIN(detail) - -inline void keep_alive_impl(handle nurse, handle patient) { - if (!nurse || !patient) +PYBIND11_NOINLINE void keep_alive_impl(handle nurse, handle patient) { + if (!nurse || !patient) { pybind11_fail("Could not activate keep_alive!"); + } - if (patient.is_none() || nurse.is_none()) + if (patient.is_none() || nurse.is_none()) { return; /* Nothing to keep alive or nothing to be kept alive by */ + } auto tinfo = all_type_info(Py_TYPE(nurse.ptr())); if (!tinfo.empty()) { /* It's a pybind-registered type, so we can store the patient in the * internal list. */ add_patient(nurse.ptr(), patient.ptr()); - } - else { + } else { /* Fall back to clever approach based on weak references taken from * Boost.Python. This is not used for pybind-registered types because * the objects can be destroyed out-of-order in a GC pass. */ - cpp_function disable_lifesupport( - [patient](handle weakref) { patient.dec_ref(); weakref.dec_ref(); }); + cpp_function disable_lifesupport([patient](handle weakref) { + patient.dec_ref(); + weakref.dec_ref(); + }); weakref wr(nurse, disable_lifesupport); @@ -1844,153 +2251,281 @@ inline void keep_alive_impl(handle nurse, handle patient) { } } -PYBIND11_NOINLINE inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { +PYBIND11_NOINLINE void +keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { auto get_arg = [&](size_t n) { - if (n == 0) + if (n == 0) { return ret; - else if (n == 1 && call.init_self) + } + if (n == 1 && call.init_self) { return call.init_self; - else if (n <= call.args.size()) + } + if (n <= call.args.size()) { return call.args[n - 1]; + } return handle(); }; keep_alive_impl(get_arg(Nurse), get_arg(Patient)); } -inline std::pair<decltype(internals::registered_types_py)::iterator, bool> all_type_info_get_cache(PyTypeObject *type) { - auto res = get_internals().registered_types_py +inline std::pair<decltype(internals::registered_types_py)::iterator, bool> +all_type_info_get_cache(PyTypeObject *type) { + auto res = get_internals() + .registered_types_py #ifdef __cpp_lib_unordered_map_try_emplace - .try_emplace(type); + .try_emplace(type); #else - .emplace(type, std::vector<detail::type_info *>()); + .emplace(type, std::vector<detail::type_info *>()); #endif if (res.second) { // New cache entry created; set up a weak reference to automatically remove it if the type // gets destroyed: weakref((PyObject *) type, cpp_function([type](handle wr) { - get_internals().registered_types_py.erase(type); - wr.dec_ref(); - })).release(); + get_internals().registered_types_py.erase(type); + + // TODO consolidate the erasure code in pybind11_meta_dealloc() in class.h + auto &cache = get_internals().inactive_override_cache; + for (auto it = cache.begin(), last = cache.end(); it != last;) { + if (it->first == reinterpret_cast<PyObject *>(type)) { + it = cache.erase(it); + } else { + ++it; + } + } + + wr.dec_ref(); + })) + .release(); } return res; } -template <typename Iterator, typename Sentinel, bool KeyIterator, return_value_policy Policy> +/* There are a large number of apparently unused template arguments because + * each combination requires a separate py::class_ registration. + */ +template <typename Access, + return_value_policy Policy, + typename Iterator, + typename Sentinel, + typename ValueType, + typename... Extra> struct iterator_state { Iterator it; Sentinel end; bool first_or_done; }; -PYBIND11_NAMESPACE_END(detail) +// Note: these helpers take the iterator by non-const reference because some +// iterators in the wild can't be dereferenced when const. The & after Iterator +// is required for MSVC < 16.9. SFINAE cannot be reused for result_type due to +// bugs in ICC, NVCC, and PGI compilers. See PR #3293. +template <typename Iterator, typename SFINAE = decltype(*std::declval<Iterator &>())> +struct iterator_access { + using result_type = decltype(*std::declval<Iterator &>()); + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 + result_type operator()(Iterator &it) const { return *it; } +}; -/// Makes a python iterator from a first and past-the-end C++ InputIterator. -template <return_value_policy Policy = return_value_policy::reference_internal, +template <typename Iterator, typename SFINAE = decltype((*std::declval<Iterator &>()).first)> +class iterator_key_access { +private: + using pair_type = decltype(*std::declval<Iterator &>()); + +public: + /* If either the pair itself or the element of the pair is a reference, we + * want to return a reference, otherwise a value. When the decltype + * expression is parenthesized it is based on the value category of the + * expression; otherwise it is the declared type of the pair member. + * The use of declval<pair_type> in the second branch rather than directly + * using *std::declval<Iterator &>() is a workaround for nvcc + * (it's not used in the first branch because going via decltype and back + * through declval does not perfectly preserve references). + */ + using result_type + = conditional_t<std::is_reference<decltype(*std::declval<Iterator &>())>::value, + decltype(((*std::declval<Iterator &>()).first)), + decltype(std::declval<pair_type>().first)>; + result_type operator()(Iterator &it) const { return (*it).first; } +}; + +template <typename Iterator, typename SFINAE = decltype((*std::declval<Iterator &>()).second)> +class iterator_value_access { +private: + using pair_type = decltype(*std::declval<Iterator &>()); + +public: + using result_type + = conditional_t<std::is_reference<decltype(*std::declval<Iterator &>())>::value, + decltype(((*std::declval<Iterator &>()).second)), + decltype(std::declval<pair_type>().second)>; + result_type operator()(Iterator &it) const { return (*it).second; } +}; + +template <typename Access, + return_value_policy Policy, typename Iterator, typename Sentinel, - typename ValueType = decltype(*std::declval<Iterator>()), + typename ValueType, typename... Extra> -iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) { - using state = detail::iterator_state<Iterator, Sentinel, false, Policy>; +iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { + using state = detail::iterator_state<Access, Policy, Iterator, Sentinel, ValueType, Extra...>; + // TODO: state captures only the types of Extra, not the values if (!detail::get_type_info(typeid(state), false)) { class_<state>(handle(), "iterator", pybind11::module_local()) - .def("__iter__", [](state &s) -> state& { return s; }) - .def("__next__", [](state &s) -> ValueType { - if (!s.first_or_done) - ++s.it; - else - s.first_or_done = false; - if (s.it == s.end) { - s.first_or_done = true; - throw stop_iteration(); - } - return *s.it; - }, std::forward<Extra>(extra)..., Policy); + .def("__iter__", [](state &s) -> state & { return s; }) + .def( + "__next__", + [](state &s) -> ValueType { + if (!s.first_or_done) { + ++s.it; + } else { + s.first_or_done = false; + } + if (s.it == s.end) { + s.first_or_done = true; + throw stop_iteration(); + } + return Access()(s.it); + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 + }, + std::forward<Extra>(extra)..., + Policy); } return cast(state{first, last, true}); } -/// Makes an python iterator over the keys (`.first`) of a iterator over pairs from a -/// first and past-the-end InputIterator. +PYBIND11_NAMESPACE_END(detail) + +/// Makes a python iterator from a first and past-the-end C++ InputIterator. template <return_value_policy Policy = return_value_policy::reference_internal, typename Iterator, typename Sentinel, - typename KeyType = decltype((*std::declval<Iterator>()).first), + typename ValueType = typename detail::iterator_access<Iterator>::result_type, typename... Extra> -iterator make_key_iterator(Iterator first, Sentinel last, Extra &&... extra) { - using state = detail::iterator_state<Iterator, Sentinel, true, Policy>; +iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { + return detail::make_iterator_impl<detail::iterator_access<Iterator>, + Policy, + Iterator, + Sentinel, + ValueType, + Extra...>(first, last, std::forward<Extra>(extra)...); +} - if (!detail::get_type_info(typeid(state), false)) { - class_<state>(handle(), "iterator", pybind11::module_local()) - .def("__iter__", [](state &s) -> state& { return s; }) - .def("__next__", [](state &s) -> KeyType { - if (!s.first_or_done) - ++s.it; - else - s.first_or_done = false; - if (s.it == s.end) { - s.first_or_done = true; - throw stop_iteration(); - } - return (*s.it).first; - }, std::forward<Extra>(extra)..., Policy); - } +/// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a +/// first and past-the-end InputIterator. +template <return_value_policy Policy = return_value_policy::reference_internal, + typename Iterator, + typename Sentinel, + typename KeyType = typename detail::iterator_key_access<Iterator>::result_type, + typename... Extra> +iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { + return detail::make_iterator_impl<detail::iterator_key_access<Iterator>, + Policy, + Iterator, + Sentinel, + KeyType, + Extra...>(first, last, std::forward<Extra>(extra)...); +} - return cast(state{first, last, true}); +/// Makes a python iterator over the values (`.second`) of a iterator over pairs from a +/// first and past-the-end InputIterator. +template <return_value_policy Policy = return_value_policy::reference_internal, + typename Iterator, + typename Sentinel, + typename ValueType = typename detail::iterator_value_access<Iterator>::result_type, + typename... Extra> +iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { + return detail::make_iterator_impl<detail::iterator_value_access<Iterator>, + Policy, + Iterator, + Sentinel, + ValueType, + Extra...>(first, last, std::forward<Extra>(extra)...); } /// Makes an iterator over values of an stl container or other container supporting /// `std::begin()`/`std::end()` template <return_value_policy Policy = return_value_policy::reference_internal, - typename Type, typename... Extra> iterator make_iterator(Type &value, Extra&&... extra) { - return make_iterator<Policy>(std::begin(value), std::end(value), extra...); + typename Type, + typename... Extra> +iterator make_iterator(Type &value, Extra &&...extra) { + return make_iterator<Policy>( + std::begin(value), std::end(value), std::forward<Extra>(extra)...); } /// Makes an iterator over the keys (`.first`) of a stl map-like container supporting /// `std::begin()`/`std::end()` template <return_value_policy Policy = return_value_policy::reference_internal, - typename Type, typename... Extra> iterator make_key_iterator(Type &value, Extra&&... extra) { - return make_key_iterator<Policy>(std::begin(value), std::end(value), extra...); + typename Type, + typename... Extra> +iterator make_key_iterator(Type &value, Extra &&...extra) { + return make_key_iterator<Policy>( + std::begin(value), std::end(value), std::forward<Extra>(extra)...); +} + +/// Makes an iterator over the values (`.second`) of a stl map-like container supporting +/// `std::begin()`/`std::end()` +template <return_value_policy Policy = return_value_policy::reference_internal, + typename Type, + typename... Extra> +iterator make_value_iterator(Type &value, Extra &&...extra) { + return make_value_iterator<Policy>( + std::begin(value), std::end(value), std::forward<Extra>(extra)...); } -template <typename InputType, typename OutputType> void implicitly_convertible() { +template <typename InputType, typename OutputType> +void implicitly_convertible() { struct set_flag { bool &flag; - set_flag(bool &flag_) : flag(flag_) { flag_ = true; } + explicit set_flag(bool &flag_) : flag(flag_) { flag_ = true; } ~set_flag() { flag = false; } }; auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * { static bool currently_used = false; - if (currently_used) // implicit conversions are non-reentrant + if (currently_used) { // implicit conversions are non-reentrant return nullptr; + } set_flag flag_helper(currently_used); - if (!detail::make_caster<InputType>().load(obj, false)) + if (!detail::make_caster<InputType>().load(obj, false)) { return nullptr; + } tuple args(1); args[0] = obj; PyObject *result = PyObject_Call((PyObject *) type, args.ptr(), nullptr); - if (result == nullptr) + if (result == nullptr) { PyErr_Clear(); + } return result; }; - if (auto tinfo = detail::get_type_info(typeid(OutputType))) - tinfo->implicit_conversions.push_back(implicit_caster); - else + if (auto *tinfo = detail::get_type_info(typeid(OutputType))) { + tinfo->implicit_conversions.emplace_back(std::move(implicit_caster)); + } else { pybind11_fail("implicitly_convertible: Unable to find type " + type_id<OutputType>()); + } } -template <typename ExceptionTranslator> -void register_exception_translator(ExceptionTranslator&& translator) { +inline void register_exception_translator(ExceptionTranslator &&translator) { detail::get_internals().registered_exception_translators.push_front( std::forward<ExceptionTranslator>(translator)); } /** + * Add a new module-local exception translator. Locally registered functions + * will be tried before any globally registered exception translators, which + * will only be invoked if the module-local handlers do not deal with + * the exception. + */ +inline void register_local_exception_translator(ExceptionTranslator &&translator) { + detail::get_local_internals().registered_exception_translators.push_front( + std::forward<ExceptionTranslator>(translator)); +} + +/** * Wrapper to generate a new Python exception type. * * This should only be used with PyErr_SetString for now. @@ -2002,19 +2537,19 @@ class exception : public object { public: exception() = default; exception(handle scope, const char *name, handle base = PyExc_Exception) { - std::string full_name = scope.attr("__name__").cast<std::string>() + - std::string(".") + name; - m_ptr = PyErr_NewException(const_cast<char *>(full_name.c_str()), base.ptr(), NULL); - if (hasattr(scope, "__dict__") && scope.attr("__dict__").contains(name)) + std::string full_name + = scope.attr("__name__").cast<std::string>() + std::string(".") + name; + m_ptr = PyErr_NewException(const_cast<char *>(full_name.c_str()), base.ptr(), nullptr); + if (hasattr(scope, "__dict__") && scope.attr("__dict__").contains(name)) { pybind11_fail("Error during initialization: multiple incompatible " - "definitions with name \"" + std::string(name) + "\""); + "definitions with name \"" + + std::string(name) + "\""); + } scope.attr(name) = *this; } // Sets the current python exception to this exception object with the given message - void operator()(const char *message) { - PyErr_SetString(m_ptr, message); - } + void operator()(const char *message) { PyErr_SetString(m_ptr, message); } }; PYBIND11_NAMESPACE_BEGIN(detail) @@ -2022,24 +2557,27 @@ PYBIND11_NAMESPACE_BEGIN(detail) // register_exception approach below. (It would be simpler to have the static local variable // directly in register_exception, but that makes clang <3.5 segfault - issue #1349). template <typename CppException> -exception<CppException> &get_exception_object() { static exception<CppException> ex; return ex; } -PYBIND11_NAMESPACE_END(detail) +exception<CppException> &get_exception_object() { + static exception<CppException> ex; + return ex; +} -/** - * Registers a Python exception in `m` of the given `name` and installs an exception translator to - * translate the C++ exception to the created Python exception using the exceptions what() method. - * This is intended for simple exception translations; for more complex translation, register the - * exception object and translator directly. - */ +// Helper function for register_exception and register_local_exception template <typename CppException> -exception<CppException> ®ister_exception(handle scope, - const char *name, - handle base = PyExc_Exception) { +exception<CppException> & +register_exception_impl(handle scope, const char *name, handle base, bool isLocal) { auto &ex = detail::get_exception_object<CppException>(); - if (!ex) ex = exception<CppException>(scope, name, base); + if (!ex) { + ex = exception<CppException>(scope, name, base); + } - register_exception_translator([](std::exception_ptr p) { - if (!p) return; + auto register_func + = isLocal ? ®ister_local_exception_translator : ®ister_exception_translator; + + register_func([](std::exception_ptr p) { + if (!p) { + return; + } try { std::rethrow_exception(p); } catch (const CppException &e) { @@ -2049,14 +2587,42 @@ exception<CppException> ®ister_exception(handle scope, return ex; } +PYBIND11_NAMESPACE_END(detail) + +/** + * Registers a Python exception in `m` of the given `name` and installs a translator to + * translate the C++ exception to the created Python exception using the what() method. + * This is intended for simple exception translations; for more complex translation, register the + * exception object and translator directly. + */ +template <typename CppException> +exception<CppException> & +register_exception(handle scope, const char *name, handle base = PyExc_Exception) { + return detail::register_exception_impl<CppException>(scope, name, base, false /* isLocal */); +} + +/** + * Registers a Python exception in `m` of the given `name` and installs a translator to + * translate the C++ exception to the created Python exception using the what() method. + * This translator will only be used for exceptions that are thrown in this module and will be + * tried before global exception translators, including those registered with register_exception. + * This is intended for simple exception translations; for more complex translation, register the + * exception object and translator directly. + */ +template <typename CppException> +exception<CppException> & +register_local_exception(handle scope, const char *name, handle base = PyExc_Exception) { + return detail::register_exception_impl<CppException>(scope, name, base, true /* isLocal */); +} + PYBIND11_NAMESPACE_BEGIN(detail) -PYBIND11_NOINLINE inline void print(tuple args, dict kwargs) { +PYBIND11_NOINLINE void print(const tuple &args, const dict &kwargs) { auto strings = tuple(args.size()); for (size_t i = 0; i < args.size(); ++i) { strings[i] = str(args[i]); } - auto sep = kwargs.contains("sep") ? kwargs["sep"] : cast(" "); - auto line = sep.attr("join")(strings); + auto sep = kwargs.contains("sep") ? kwargs["sep"] : str(" "); + auto line = sep.attr("join")(std::move(strings)); object file; if (kwargs.contains("file")) { @@ -2074,11 +2640,12 @@ PYBIND11_NOINLINE inline void print(tuple args, dict kwargs) { } auto write = file.attr("write"); - write(line); - write(kwargs.contains("end") ? kwargs["end"] : cast("\n")); + write(std::move(line)); + write(kwargs.contains("end") ? kwargs["end"] : str("\n")); - if (kwargs.contains("flush") && kwargs["flush"].cast<bool>()) + if (kwargs.contains("flush") && kwargs["flush"].cast<bool>()) { file.attr("flush")(); + } } PYBIND11_NAMESPACE_END(detail) @@ -2088,236 +2655,106 @@ void print(Args &&...args) { detail::print(c.args(), c.kwargs()); } -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) - -/* The functions below essentially reproduce the PyGILState_* API using a RAII - * pattern, but there are a few important differences: - * - * 1. When acquiring the GIL from an non-main thread during the finalization - * phase, the GILState API blindly terminates the calling thread, which - * is often not what is wanted. This API does not do this. - * - * 2. The gil_scoped_release function can optionally cut the relationship - * of a PyThreadState and its associated thread, which allows moving it to - * another thread (this is a fairly rare/advanced use case). - * - * 3. The reference count of an acquired thread state can be controlled. This - * can be handy to prevent cases where callbacks issued from an external - * thread would otherwise constantly construct and destroy thread state data - * structures. - * - * See the Python bindings of NanoGUI (http://github.com/wjakob/nanogui) for an - * example which uses features 2 and 3 to migrate the Python thread of - * execution to another thread (to run the event loop on the original thread, - * in this case). - */ - -class gil_scoped_acquire { -public: - PYBIND11_NOINLINE gil_scoped_acquire() { - auto const &internals = detail::get_internals(); - tstate = (PyThreadState *) PYBIND11_TLS_GET_VALUE(internals.tstate); - - if (!tstate) { - /* Check if the GIL was acquired using the PyGILState_* API instead (e.g. if - calling from a Python thread). Since we use a different key, this ensures - we don't create a new thread state and deadlock in PyEval_AcquireThread - below. Note we don't save this state with internals.tstate, since we don't - create it we would fail to clear it (its reference count should be > 0). */ - tstate = PyGILState_GetThisThreadState(); - } - - if (!tstate) { - tstate = PyThreadState_New(internals.istate); - #if !defined(NDEBUG) - if (!tstate) - pybind11_fail("scoped_acquire: could not create thread state!"); - #endif - tstate->gilstate_counter = 0; - PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); - } else { - release = detail::get_thread_state_unchecked() != tstate; - } - - if (release) { - PyEval_AcquireThread(tstate); - } - - inc_ref(); - } - - void inc_ref() { - ++tstate->gilstate_counter; - } - - PYBIND11_NOINLINE void dec_ref() { - --tstate->gilstate_counter; - #if !defined(NDEBUG) - if (detail::get_thread_state_unchecked() != tstate) - pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); - if (tstate->gilstate_counter < 0) - pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); - #endif - if (tstate->gilstate_counter == 0) { - #if !defined(NDEBUG) - if (!release) - pybind11_fail("scoped_acquire::dec_ref(): internal error!"); - #endif - PyThreadState_Clear(tstate); - if (active) - PyThreadState_DeleteCurrent(); - PYBIND11_TLS_DELETE_VALUE(detail::get_internals().tstate); - release = false; - } - } - - /// This method will disable the PyThreadState_DeleteCurrent call and the - /// GIL won't be acquired. This method should be used if the interpreter - /// could be shutting down when this is called, as thread deletion is not - /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and - /// protect subsequent code. - PYBIND11_NOINLINE void disarm() { - active = false; - } - - PYBIND11_NOINLINE ~gil_scoped_acquire() { - dec_ref(); - if (release) - PyEval_SaveThread(); - } -private: - PyThreadState *tstate = nullptr; - bool release = true; - bool active = true; -}; - -class gil_scoped_release { -public: - explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { - // `get_internals()` must be called here unconditionally in order to initialize - // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an - // initialization race could occur as multiple threads try `gil_scoped_acquire`. - const auto &internals = detail::get_internals(); - tstate = PyEval_SaveThread(); - if (disassoc) { - auto key = internals.tstate; - PYBIND11_TLS_DELETE_VALUE(key); - } - } - - /// This method will disable the PyThreadState_DeleteCurrent call and the - /// GIL won't be acquired. This method should be used if the interpreter - /// could be shutting down when this is called, as thread deletion is not - /// allowed during shutdown. Check _Py_IsFinalizing() on Python 3.7+, and - /// protect subsequent code. - PYBIND11_NOINLINE void disarm() { - active = false; - } - - ~gil_scoped_release() { - if (!tstate) - return; - // `PyEval_RestoreThread()` should not be called if runtime is finalizing - if (active) - PyEval_RestoreThread(tstate); - if (disassoc) { - auto key = detail::get_internals().tstate; - PYBIND11_TLS_REPLACE_VALUE(key, tstate); - } - } -private: - PyThreadState *tstate; - bool disassoc; - bool active = true; -}; -#elif defined(PYPY_VERSION) -class gil_scoped_acquire { - PyGILState_STATE state; -public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } - ~gil_scoped_acquire() { PyGILState_Release(state); } - void disarm() {} -}; - -class gil_scoped_release { - PyThreadState *state; -public: - gil_scoped_release() { state = PyEval_SaveThread(); } - ~gil_scoped_release() { PyEval_RestoreThread(state); } - void disarm() {} -}; -#else -class gil_scoped_acquire { - void disarm() {} -}; -class gil_scoped_release { - void disarm() {} -}; -#endif +inline void +error_already_set::m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr) { + gil_scoped_acquire gil; + error_scope scope; + delete raw_ptr; +} -error_already_set::~error_already_set() { - if (m_type) { - gil_scoped_acquire gil; - error_scope scope; - m_type.release().dec_ref(); - m_value.release().dec_ref(); - m_trace.release().dec_ref(); - } +inline const char *error_already_set::what() const noexcept { + gil_scoped_acquire gil; + error_scope scope; + return m_fetched_error->error_string().c_str(); } PYBIND11_NAMESPACE_BEGIN(detail) -inline function get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { + +inline function +get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { handle self = get_object_handle(this_ptr, this_type); - if (!self) + if (!self) { return function(); + } handle type = type::handle_of(self); auto key = std::make_pair(type.ptr(), name); /* Cache functions that aren't overridden in Python to avoid many costly Python dictionary lookups below */ auto &cache = get_internals().inactive_override_cache; - if (cache.find(key) != cache.end()) + if (cache.find(key) != cache.end()) { return function(); + } function override = getattr(self, name, function()); if (override.is_cpp_function()) { - cache.insert(key); + cache.insert(std::move(key)); return function(); } /* Don't call dispatch code if invoked from overridden function. Unfortunately this doesn't work on PyPy. */ #if !defined(PYPY_VERSION) +# if PY_VERSION_HEX >= 0x03090000 + PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); + if (frame != nullptr) { + PyCodeObject *f_code = PyFrame_GetCode(frame); + // f_code is guaranteed to not be NULL + if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) { + PyObject *locals = PyEval_GetLocals(); + if (locals != nullptr) { + PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames"); + PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0); + Py_DECREF(co_varnames); + PyObject *self_caller = dict_getitem(locals, self_arg); + if (self_caller == self.ptr()) { + Py_DECREF(f_code); + Py_DECREF(frame); + return function(); + } + } + } + Py_DECREF(f_code); + Py_DECREF(frame); + } +# else PyFrameObject *frame = PyThreadState_Get()->frame; - if (frame && (std::string) str(frame->f_code->co_name) == name && - frame->f_code->co_argcount > 0) { + if (frame != nullptr && (std::string) str(frame->f_code->co_name) == name + && frame->f_code->co_argcount > 0) { PyFrame_FastToLocals(frame); - PyObject *self_caller = PyDict_GetItem( - frame->f_locals, PyTuple_GET_ITEM(frame->f_code->co_varnames, 0)); - if (self_caller == self.ptr()) + PyObject *self_caller + = dict_getitem(frame->f_locals, PyTuple_GET_ITEM(frame->f_code->co_varnames, 0)); + if (self_caller == self.ptr()) { return function(); + } } +# endif + #else /* PyPy currently doesn't provide a detailed cpyext emulation of frame objects, so we have to emulate this using Python. This is going to be slow..*/ - dict d; d["self"] = self; d["name"] = pybind11::str(name); - PyObject *result = PyRun_String( - "import inspect\n" - "frame = inspect.currentframe()\n" - "if frame is not None:\n" - " frame = frame.f_back\n" - " if frame is not None and str(frame.f_code.co_name) == name and " - "frame.f_code.co_argcount > 0:\n" - " self_caller = frame.f_locals[frame.f_code.co_varnames[0]]\n" - " if self_caller == self:\n" - " self = None\n", - Py_file_input, d.ptr(), d.ptr()); + dict d; + d["self"] = self; + d["name"] = pybind11::str(name); + PyObject *result + = PyRun_String("import inspect\n" + "frame = inspect.currentframe()\n" + "if frame is not None:\n" + " frame = frame.f_back\n" + " if frame is not None and str(frame.f_code.co_name) == name and " + "frame.f_code.co_argcount > 0:\n" + " self_caller = frame.f_locals[frame.f_code.co_varnames[0]]\n" + " if self_caller == self:\n" + " self = None\n", + Py_file_input, + d.ptr(), + d.ptr()); if (result == nullptr) throw error_already_set(); + Py_DECREF(result); if (d["self"].is_none()) return function(); - Py_DECREF(result); #endif return override; @@ -2325,36 +2762,40 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty PYBIND11_NAMESPACE_END(detail) /** \rst - Try to retrieve a python method by the provided name from the instance pointed to by the this_ptr. + Try to retrieve a python method by the provided name from the instance pointed to by the + this_ptr. - :this_ptr: The pointer to the object the overridden method should be retrieved for. This should be - the first non-trampoline class encountered in the inheritance chain. + :this_ptr: The pointer to the object the overridden method should be retrieved for. This should + be the first non-trampoline class encountered in the inheritance chain. :name: The name of the overridden Python method to retrieve. :return: The Python method by this name from the object or an empty function wrapper. \endrst */ -template <class T> function get_override(const T *this_ptr, const char *name) { - auto tinfo = detail::get_type_info(typeid(T)); +template <class T> +function get_override(const T *this_ptr, const char *name) { + auto *tinfo = detail::get_type_info(typeid(T)); return tinfo ? detail::get_type_override(this_ptr, tinfo, name) : function(); } -#define PYBIND11_OVERRIDE_IMPL(ret_type, cname, name, ...) \ - do { \ - pybind11::gil_scoped_acquire gil; \ - pybind11::function override = pybind11::get_override(static_cast<const cname *>(this), name); \ - if (override) { \ - auto o = override(__VA_ARGS__); \ - if (pybind11::detail::cast_is_temporary_value_reference<ret_type>::value) { \ - static pybind11::detail::override_caster_t<ret_type> caster; \ - return pybind11::detail::cast_ref<ret_type>(std::move(o), caster); \ - } \ - else return pybind11::detail::cast_safe<ret_type>(std::move(o)); \ - } \ +#define PYBIND11_OVERRIDE_IMPL(ret_type, cname, name, ...) \ + do { \ + pybind11::gil_scoped_acquire gil; \ + pybind11::function override \ + = pybind11::get_override(static_cast<const cname *>(this), name); \ + if (override) { \ + auto o = override(__VA_ARGS__); \ + if (pybind11::detail::cast_is_temporary_value_reference<ret_type>::value) { \ + static pybind11::detail::override_caster_t<ret_type> caster; \ + return pybind11::detail::cast_ref<ret_type>(std::move(o), caster); \ + } \ + return pybind11::detail::cast_safe<ret_type>(std::move(o)); \ + } \ } while (false) /** \rst - Macro to populate the virtual method in the trampoline class. This macro tries to look up a method named 'fn' - from the Python side, deals with the :ref:`gil` and necessary argument conversions to call this method and return - the appropriate type. See :ref:`overriding_virtuals` for more information. This macro should be used when the method + Macro to populate the virtual method in the trampoline class. This macro tries to look up a + method named 'fn' from the Python side, deals with the :ref:`gil` and necessary argument + conversions to call this method and return the appropriate type. + See :ref:`overriding_virtuals` for more information. This macro should be used when the method name in C is not the same as the method name in Python. For example with `__str__`. .. code-block:: cpp @@ -2368,26 +2809,28 @@ template <class T> function get_override(const T *this_ptr, const char *name) { ); } \endrst */ -#define PYBIND11_OVERRIDE_NAME(ret_type, cname, name, fn, ...) \ - do { \ +#define PYBIND11_OVERRIDE_NAME(ret_type, cname, name, fn, ...) \ + do { \ PYBIND11_OVERRIDE_IMPL(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__); \ - return cname::fn(__VA_ARGS__); \ + return cname::fn(__VA_ARGS__); \ } while (false) /** \rst - Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERRIDE_NAME`, except that it - throws if no override can be found. + Macro for pure virtual functions, this function is identical to + :c:macro:`PYBIND11_OVERRIDE_NAME`, except that it throws if no override can be found. \endrst */ -#define PYBIND11_OVERRIDE_PURE_NAME(ret_type, cname, name, fn, ...) \ - do { \ +#define PYBIND11_OVERRIDE_PURE_NAME(ret_type, cname, name, fn, ...) \ + do { \ PYBIND11_OVERRIDE_IMPL(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__); \ - pybind11::pybind11_fail("Tried to call pure virtual function \"" PYBIND11_STRINGIFY(cname) "::" name "\""); \ + pybind11::pybind11_fail( \ + "Tried to call pure virtual function \"" PYBIND11_STRINGIFY(cname) "::" name "\""); \ } while (false) /** \rst - Macro to populate the virtual method in the trampoline class. This macro tries to look up the method - from the Python side, deals with the :ref:`gil` and necessary argument conversions to call this method and return - the appropriate type. This macro should be used if the method name in C and in Python are identical. + Macro to populate the virtual method in the trampoline class. This macro tries to look up the + method from the Python side, deals with the :ref:`gil` and necessary argument conversions to + call this method and return the appropriate type. This macro should be used if the method name + in C and in Python are identical. See :ref:`overriding_virtuals` for more information. .. code-block:: cpp @@ -2408,21 +2851,22 @@ template <class T> function get_override(const T *this_ptr, const char *name) { } }; \endrst */ -#define PYBIND11_OVERRIDE(ret_type, cname, fn, ...) \ +#define PYBIND11_OVERRIDE(ret_type, cname, fn, ...) \ PYBIND11_OVERRIDE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) /** \rst - Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERRIDE`, except that it throws - if no override can be found. + Macro for pure virtual functions, this function is identical to :c:macro:`PYBIND11_OVERRIDE`, + except that it throws if no override can be found. \endrst */ -#define PYBIND11_OVERRIDE_PURE(ret_type, cname, fn, ...) \ - PYBIND11_OVERRIDE_PURE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) - +#define PYBIND11_OVERRIDE_PURE(ret_type, cname, fn, ...) \ + PYBIND11_OVERRIDE_PURE_NAME( \ + PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), #fn, fn, __VA_ARGS__) // Deprecated versions PYBIND11_DEPRECATED("get_type_overload has been deprecated") -inline function get_type_overload(const void *this_ptr, const detail::type_info *this_type, const char *name) { +inline function +get_type_overload(const void *this_ptr, const detail::type_info *this_type, const char *name) { return detail::get_type_override(this_ptr, this_type, name); } @@ -2431,21 +2875,16 @@ inline function get_overload(const T *this_ptr, const char *name) { return get_override(this_ptr, name); } -#define PYBIND11_OVERLOAD_INT(ret_type, cname, name, ...) \ +#define PYBIND11_OVERLOAD_INT(ret_type, cname, name, ...) \ PYBIND11_OVERRIDE_IMPL(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, __VA_ARGS__) -#define PYBIND11_OVERLOAD_NAME(ret_type, cname, name, fn, ...) \ +#define PYBIND11_OVERLOAD_NAME(ret_type, cname, name, fn, ...) \ PYBIND11_OVERRIDE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, fn, __VA_ARGS__) -#define PYBIND11_OVERLOAD_PURE_NAME(ret_type, cname, name, fn, ...) \ - PYBIND11_OVERRIDE_PURE_NAME(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, fn, __VA_ARGS__); -#define PYBIND11_OVERLOAD(ret_type, cname, fn, ...) \ +#define PYBIND11_OVERLOAD_PURE_NAME(ret_type, cname, name, fn, ...) \ + PYBIND11_OVERRIDE_PURE_NAME( \ + PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), name, fn, __VA_ARGS__); +#define PYBIND11_OVERLOAD(ret_type, cname, fn, ...) \ PYBIND11_OVERRIDE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__) -#define PYBIND11_OVERLOAD_PURE(ret_type, cname, fn, ...) \ +#define PYBIND11_OVERLOAD_PURE(ret_type, cname, fn, ...) \ PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma warning(pop) -#elif defined(__GNUG__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 78db7947..c93e3d3b 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -11,30 +11,53 @@ #include "detail/common.h" #include "buffer_info.h" -#include <utility> + +#include <assert.h> +#include <cstddef> +#include <exception> +#include <frameobject.h> +#include <iterator> +#include <memory> +#include <string> #include <type_traits> +#include <typeinfo> +#include <utility> + +#if defined(PYBIND11_HAS_OPTIONAL) +# include <optional> +#endif + +#ifdef PYBIND11_HAS_STRING_VIEW +# include <string_view> +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + /* A few forward declarations */ -class handle; class object; -class str; class iterator; +class handle; +class object; +class str; +class iterator; class type; -struct arg; struct arg_v; +struct arg; +struct arg_v; PYBIND11_NAMESPACE_BEGIN(detail) class args_proxy; -inline bool isinstance_generic(handle obj, const std::type_info &tp); +bool isinstance_generic(handle obj, const std::type_info &tp); // Accessor forward declarations -template <typename Policy> class accessor; +template <typename Policy> +class accessor; namespace accessor_policies { - struct obj_attr; - struct str_attr; - struct generic_item; - struct sequence_item; - struct list_item; - struct tuple_item; +struct obj_attr; +struct str_attr; +struct generic_item; +struct sequence_item; +struct list_item; +struct tuple_item; } // namespace accessor_policies using obj_attr_accessor = accessor<accessor_policies::obj_attr>; using str_attr_accessor = accessor<accessor_policies::str_attr>; @@ -44,8 +67,9 @@ using list_accessor = accessor<accessor_policies::list_item>; using tuple_accessor = accessor<accessor_policies::tuple_item>; /// Tag and check to identify a class which implements the Python object API -class pyobject_tag { }; -template <typename T> using is_pyobject = std::is_base_of<pyobject_tag, remove_reference_t<T>>; +class pyobject_tag {}; +template <typename T> +using is_pyobject = std::is_base_of<pyobject_tag, remove_reference_t<T>>; /** \rst A mixin class which adds common functions to `handle`, `object` and various accessors. @@ -71,7 +95,9 @@ public: or `object` subclass causes a call to ``__setitem__``. \endrst */ item_accessor operator[](handle key) const; - /// See above (the only difference is that they key is provided as a string literal) + /// See above (the only difference is that the key's reference is stolen) + item_accessor operator[](object &&key) const; + /// See above (the only difference is that the key is provided as a string literal) item_accessor operator[](const char *key) const; /** \rst @@ -81,7 +107,9 @@ public: or `object` subclass causes a call to ``setattr``. \endrst */ obj_attr_accessor attr(handle key) const; - /// See above (the only difference is that they key is provided as a string literal) + /// See above (the only difference is that the key's reference is stolen) + obj_attr_accessor attr(object &&key) const; + /// See above (the only difference is that the key is provided as a string literal) str_attr_accessor attr(const char *key) const; /** \rst @@ -93,7 +121,8 @@ public: args_proxy operator*() const; /// Check if the given item is contained within this object, i.e. ``item in obj``. - template <typename T> bool contains(T &&item) const; + template <typename T> + bool contains(T &&item) const; /** \rst Assuming the Python object is a function or implements the ``__call__`` @@ -105,44 +134,46 @@ public: function will throw a `cast_error` exception. When the Python function call fails, a `error_already_set` exception is thrown. \endrst */ - template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args> + template <return_value_policy policy = return_value_policy::automatic_reference, + typename... Args> object operator()(Args &&...args) const; - template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args> + template <return_value_policy policy = return_value_policy::automatic_reference, + typename... Args> PYBIND11_DEPRECATED("call(...) was deprecated in favor of operator()(...)") - object call(Args&&... args) const; + object call(Args &&...args) const; /// Equivalent to ``obj is other`` in Python. - bool is(object_api const& other) const { return derived().ptr() == other.derived().ptr(); } + bool is(object_api const &other) const { return derived().ptr() == other.derived().ptr(); } /// Equivalent to ``obj is None`` in Python. bool is_none() const { return derived().ptr() == Py_None; } /// Equivalent to obj == other in Python - bool equal(object_api const &other) const { return rich_compare(other, Py_EQ); } - bool not_equal(object_api const &other) const { return rich_compare(other, Py_NE); } - bool operator<(object_api const &other) const { return rich_compare(other, Py_LT); } + bool equal(object_api const &other) const { return rich_compare(other, Py_EQ); } + bool not_equal(object_api const &other) const { return rich_compare(other, Py_NE); } + bool operator<(object_api const &other) const { return rich_compare(other, Py_LT); } bool operator<=(object_api const &other) const { return rich_compare(other, Py_LE); } - bool operator>(object_api const &other) const { return rich_compare(other, Py_GT); } + bool operator>(object_api const &other) const { return rich_compare(other, Py_GT); } bool operator>=(object_api const &other) const { return rich_compare(other, Py_GE); } object operator-() const; object operator~() const; object operator+(object_api const &other) const; - object operator+=(object_api const &other) const; + object operator+=(object_api const &other); object operator-(object_api const &other) const; - object operator-=(object_api const &other) const; + object operator-=(object_api const &other); object operator*(object_api const &other) const; - object operator*=(object_api const &other) const; + object operator*=(object_api const &other); object operator/(object_api const &other) const; - object operator/=(object_api const &other) const; + object operator/=(object_api const &other); object operator|(object_api const &other) const; - object operator|=(object_api const &other) const; + object operator|=(object_api const &other); object operator&(object_api const &other) const; - object operator&=(object_api const &other) const; + object operator&=(object_api const &other); object operator^(object_api const &other) const; - object operator^=(object_api const &other) const; + object operator^=(object_api const &other); object operator<<(object_api const &other) const; - object operator<<=(object_api const &other) const; + object operator<<=(object_api const &other); object operator>>(object_api const &other) const; - object operator>>=(object_api const &other) const; + object operator>>=(object_api const &other); PYBIND11_DEPRECATED("Use py::str(obj) instead") pybind11::str str() const; @@ -153,15 +184,25 @@ public: /// Return the object's current reference count int ref_count() const { return static_cast<int>(Py_REFCNT(derived().ptr())); } - // TODO PYBIND11_DEPRECATED("Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()") + // TODO PYBIND11_DEPRECATED( + // "Call py::type::handle_of(h) or py::type::of(h) instead of h.get_type()") handle get_type() const; private: bool rich_compare(object_api const &other, int value) const; }; +template <typename T> +using is_pyobj_ptr_or_nullptr_t = detail::any_of<std::is_same<T, PyObject *>, + std::is_same<T, PyObject *const>, + std::is_same<T, std::nullptr_t>>; + PYBIND11_NAMESPACE_END(detail) +#if !defined(PYBIND11_HANDLE_REF_DEBUG) && !defined(NDEBUG) +# define PYBIND11_HANDLE_REF_DEBUG +#endif + /** \rst Holds a reference to a Python object (no reference counting) @@ -177,8 +218,24 @@ class handle : public detail::object_api<handle> { public: /// The default constructor creates a handle with a ``nullptr``-valued pointer handle() = default; - /// Creates a ``handle`` from the given raw Python object pointer - handle(PyObject *ptr) : m_ptr(ptr) { } // Allow implicit conversion from PyObject* + + /// Enable implicit conversion from ``PyObject *`` and ``nullptr``. + /// Not using ``handle(PyObject *ptr)`` to avoid implicit conversion from ``0``. + template <typename T, + detail::enable_if_t<detail::is_pyobj_ptr_or_nullptr_t<T>::value, int> = 0> + // NOLINTNEXTLINE(google-explicit-constructor) + handle(T ptr) : m_ptr(ptr) {} + + /// Enable implicit conversion through ``T::operator PyObject *()``. + template < + typename T, + detail::enable_if_t<detail::all_of<detail::none_of<std::is_base_of<handle, T>, + detail::is_pyobj_ptr_or_nullptr_t<T>>, + std::is_convertible<T, PyObject *>>::value, + int> + = 0> + // NOLINTNEXTLINE(google-explicit-constructor) + handle(T &obj) : m_ptr(obj) {} /// Return the underlying ``PyObject *`` pointer PyObject *ptr() const { return m_ptr; } @@ -189,20 +246,40 @@ public: preferable to use the `object` class which derives from `handle` and calls this function automatically. Returns a reference to itself. \endrst */ - const handle& inc_ref() const & { Py_XINCREF(m_ptr); return *this; } + const handle &inc_ref() const & { +#ifdef PYBIND11_HANDLE_REF_DEBUG + inc_ref_counter(1); +#endif +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::inc_ref()"); + } +#endif + Py_XINCREF(m_ptr); + return *this; + } /** \rst Manually decrease the reference count of the Python object. Usually, it is preferable to use the `object` class which derives from `handle` and calls this function automatically. Returns a reference to itself. \endrst */ - const handle& dec_ref() const & { Py_XDECREF(m_ptr); return *this; } + const handle &dec_ref() const & { +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::dec_ref()"); + } +#endif + Py_XDECREF(m_ptr); + return *this; + } /** \rst Attempt to cast the Python object into the given C++ type. A `cast_error` will be throw upon failure. \endrst */ - template <typename T> T cast() const; + template <typename T> + T cast() const; /// Return ``true`` when the `handle` wraps a valid Python object explicit operator bool() const { return m_ptr != nullptr; } /** \rst @@ -215,8 +292,41 @@ public: bool operator!=(const handle &h) const { return m_ptr != h.m_ptr; } PYBIND11_DEPRECATED("Use handle::operator bool() instead") bool check() const { return m_ptr != nullptr; } + protected: PyObject *m_ptr = nullptr; + +private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n", + function_name.c_str()); + fflush(stderr); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + "The failing %s call was triggered on a %s object.\n", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + fflush(stderr); + } + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG + static std::size_t inc_ref_counter(std::size_t add) { + thread_local std::size_t counter = 0; + counter += add; + return counter; + } + +public: + static std::size_t inc_ref_counter() { return inc_ref_counter(0); } +#endif }; /** \rst @@ -233,11 +343,15 @@ class object : public handle { public: object() = default; PYBIND11_DEPRECATED("Use reinterpret_borrow<object>() or reinterpret_steal<object>()") - object(handle h, bool is_borrowed) : handle(h) { if (is_borrowed) inc_ref(); } + object(handle h, bool is_borrowed) : handle(h) { + if (is_borrowed) { + inc_ref(); + } + } /// Copy constructor; always increases the reference count object(const object &o) : handle(o) { inc_ref(); } /// Move constructor; steals the object from ``other`` and preserves its reference count - object(object &&other) noexcept { m_ptr = other.m_ptr; other.m_ptr = nullptr; } + object(object &&other) noexcept : handle(other) { other.m_ptr = nullptr; } /// Destructor; automatically calls `handle::dec_ref()` ~object() { dec_ref(); } @@ -247,19 +361,25 @@ public: Python object. \endrst */ handle release() { - PyObject *tmp = m_ptr; - m_ptr = nullptr; - return handle(tmp); + PyObject *tmp = m_ptr; + m_ptr = nullptr; + return handle(tmp); } - object& operator=(const object &other) { - other.inc_ref(); - dec_ref(); - m_ptr = other.m_ptr; + object &operator=(const object &other) { + // Skip inc_ref and dec_ref if both objects are the same + if (!this->is(other)) { + other.inc_ref(); + // Use temporary variable to ensure `*this` remains valid while + // `Py_XDECREF` executes, in case `*this` is accessible from Python. + handle temp(m_ptr); + m_ptr = other.m_ptr; + temp.dec_ref(); + } return *this; } - object& operator=(object &&other) noexcept { + object &operator=(object &&other) noexcept { if (this != &other) { handle temp(m_ptr); m_ptr = other.m_ptr; @@ -269,23 +389,43 @@ public: return *this; } +#define PYBIND11_INPLACE_OP(iop) \ + object iop(object_api const &other) { return operator=(handle::iop(other)); } + + PYBIND11_INPLACE_OP(operator+=) + PYBIND11_INPLACE_OP(operator-=) + PYBIND11_INPLACE_OP(operator*=) + PYBIND11_INPLACE_OP(operator/=) + PYBIND11_INPLACE_OP(operator|=) + PYBIND11_INPLACE_OP(operator&=) + PYBIND11_INPLACE_OP(operator^=) + PYBIND11_INPLACE_OP(operator<<=) + PYBIND11_INPLACE_OP(operator>>=) +#undef PYBIND11_INPLACE_OP + // Calling cast() on an object lvalue just copies (via handle::cast) - template <typename T> T cast() const &; + template <typename T> + T cast() const &; // Calling on an object rvalue does a move, if needed and/or possible - template <typename T> T cast() &&; + template <typename T> + T cast() &&; protected: // Tags for choosing constructors from raw PyObject * - struct borrowed_t { }; - struct stolen_t { }; + struct borrowed_t {}; + struct stolen_t {}; - template <typename T> friend T reinterpret_borrow(handle); - template <typename T> friend T reinterpret_steal(handle); + /// @cond BROKEN + template <typename T> + friend T reinterpret_borrow(handle); + template <typename T> + friend T reinterpret_steal(handle); + /// @endcond public: // Only accessible from derived classes and the reinterpret_* functions object(handle h, borrowed_t) : handle(h) { inc_ref(); } - object(handle h, stolen_t) : handle(h) { } + object(handle h, stolen_t) : handle(h) {} }; /** \rst @@ -301,7 +441,10 @@ public: // or py::tuple t = reinterpret_borrow<py::tuple>(p); // <-- `p` must be already be a `tuple` \endrst */ -template <typename T> T reinterpret_borrow(handle h) { return {h, object::borrowed_t{}}; } +template <typename T> +T reinterpret_borrow(handle h) { + return {h, object::borrowed_t{}}; +} /** \rst Like `reinterpret_borrow`, but steals the reference. @@ -311,44 +454,297 @@ template <typename T> T reinterpret_borrow(handle h) { return {h, object::borrow PyObject *p = PyObject_Str(obj); py::str s = reinterpret_steal<py::str>(p); // <-- `p` must be already be a `str` \endrst */ -template <typename T> T reinterpret_steal(handle h) { return {h, object::stolen_t{}}; } +template <typename T> +T reinterpret_steal(handle h) { + return {h, object::stolen_t{}}; +} PYBIND11_NAMESPACE_BEGIN(detail) -inline std::string error_string(); -PYBIND11_NAMESPACE_END(detail) -/// Fetch and hold an error which was already set in Python. An instance of this is typically -/// thrown to propagate python-side errors back through C++ which can either be caught manually or -/// else falls back to the function dispatcher (which then raises the captured error back to -/// python). -class error_already_set : public std::runtime_error { -public: - /// Constructs a new exception from the current Python error indicator, if any. The current - /// Python error indicator will be cleared. - error_already_set() : std::runtime_error(detail::error_string()) { +// Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). +inline const char *obj_class_name(PyObject *obj) { + if (PyType_Check(obj)) { + return reinterpret_cast<PyTypeObject *>(obj)->tp_name; + } + return Py_TYPE(obj)->tp_name; +} + +std::string error_string(); + +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. +struct error_fetch_and_normalize { + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. + explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); + if (!m_type) { + pybind11_fail("Internal error: " + std::string(called) + + " called while " + "Python error indicator not set."); + } + const char *exc_type_name_orig = detail::obj_class_name(m_type.ptr()); + if (exc_type_name_orig == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to obtain the name " + "of the original active exception type."); + } + m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else + // PyErr_NormalizeException() may change the exception type if there are cascading + // failures. This can potentially be extremely confusing. + PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); + if (m_type.ptr() == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to normalize the " + "active exception."); + } + const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); + if (exc_type_name_norm == nullptr) { + pybind11_fail("Internal error: " + std::string(called) + + " failed to obtain the name " + "of the normalized active exception type."); + } +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 + // This behavior runs the risk of masking errors in the error handling, but avoids a + // conflict with PyPy, which relies on the normalization here to change OSError to + // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). + m_lazy_error_string = exc_type_name_norm; +# else + if (exc_type_name_norm != m_lazy_error_string) { + std::string msg = std::string(called) + + ": MISMATCH of original and normalized " + "active exception types: "; + msg += "ORIGINAL "; + msg += m_lazy_error_string; + msg += " REPLACED BY "; + msg += exc_type_name_norm; + msg += ": " + format_value_and_trace(); + pybind11_fail(msg); + } +# endif +#endif + } + + error_fetch_and_normalize(const error_fetch_and_normalize &) = delete; + error_fetch_and_normalize(error_fetch_and_normalize &&) = delete; + + std::string format_value_and_trace() const { + std::string result; + std::string message_error_string; + if (m_value) { + auto value_str = reinterpret_steal<object>(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = "<MESSAGE UNAVAILABLE DUE TO ANOTHER EXCEPTION>"; + if (!value_str) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + // Not using `value_str.cast<std::string>()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal<object>( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast<std::size_t>(length)); + } + } + } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal<object>(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal<object>( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast<std::size_t>(length)); + } + } + } + } + } +#endif + } else { + result = "<MESSAGE UNAVAILABLE>"; + } + if (result.empty()) { + result = "<EMPTY MESSAGE>"; + } + + bool have_trace = false; + if (m_trace) { +#if !defined(PYPY_VERSION) + auto *tb = reinterpret_cast<PyTracebackObject *>(m_trace.ptr()); + + // Get the deepest trace possible. + while (tb->tb_next) { + tb = tb->tb_next; + } + + PyFrameObject *frame = tb->tb_frame; + Py_XINCREF(frame); + result += "\n\nAt:\n"; + while (frame) { +# if PY_VERSION_HEX >= 0x030900B1 + PyCodeObject *f_code = PyFrame_GetCode(frame); +# else + PyCodeObject *f_code = frame->f_code; + Py_INCREF(f_code); +# endif + int lineno = PyFrame_GetLineNumber(frame); + result += " "; + result += handle(f_code->co_filename).cast<std::string>(); + result += '('; + result += std::to_string(lineno); + result += "): "; + result += handle(f_code->co_name).cast<std::string>(); + result += '\n'; + Py_DECREF(f_code); +# if PY_VERSION_HEX >= 0x030900B1 + auto *b_frame = PyFrame_GetBack(frame); +# else + auto *b_frame = frame->f_back; + Py_XINCREF(b_frame); +# endif + Py_DECREF(frame); + frame = b_frame; + } + + have_trace = true; +#endif //! defined(PYPY_VERSION) + } + + if (!message_error_string.empty()) { + if (!have_trace) { + result += '\n'; + } + result += "\nMESSAGE UNAVAILABLE DUE TO EXCEPTION: " + message_error_string; + } + + return result; } - error_already_set(const error_already_set &) = default; - error_already_set(error_already_set &&) = default; + std::string const &error_string() const { + if (!m_lazy_error_string_completed) { + m_lazy_error_string += ": " + format_value_and_trace(); + m_lazy_error_string_completed = true; + } + return m_lazy_error_string; + } + + void restore() { + if (m_restore_called) { + pybind11_fail("Internal error: pybind11::detail::error_fetch_and_normalize::restore() " + "called a second time. ORIGINAL ERROR: " + + error_string()); + } + PyErr_Restore(m_type.inc_ref().ptr(), m_value.inc_ref().ptr(), m_trace.inc_ref().ptr()); + m_restore_called = true; + } + + bool matches(handle exc) const { + return (PyErr_GivenExceptionMatches(m_type.ptr(), exc.ptr()) != 0); + } + + // Not protecting these for simplicity. + object m_type, m_value, m_trace; + +private: + // Only protecting invariants. + mutable std::string m_lazy_error_string; + mutable bool m_lazy_error_string_completed = false; + mutable bool m_restore_called = false; +}; - inline ~error_already_set() override; +inline std::string error_string() { + return error_fetch_and_normalize("pybind11::detail::error_string").error_string(); +} - /// Give the currently-held error back to Python, if any. If there is currently a Python error - /// already set it is cleared first. After this call, the current object no longer stores the - /// error variables (but the `.what()` string is still available). - void restore() { PyErr_Restore(m_type.release().ptr(), m_value.release().ptr(), m_trace.release().ptr()); } +PYBIND11_NAMESPACE_END(detail) - /// If it is impossible to raise the currently-held error, such as in destructor, we can write - /// it out using Python's unraisable hook (sys.unraisablehook). The error context should be - /// some object whose repr() helps identify the location of the error. Python already knows the - /// type and value of the error, so there is no need to repeat that. For example, __func__ could - /// be helpful. After this call, the current object no longer stores the error variables, - /// and neither does Python. +/// Fetch and hold an error which was already set in Python. An instance of this is typically +/// thrown to propagate python-side errors back through C++ which can either be caught manually or +/// else falls back to the function dispatcher (which then raises the captured error back to +/// python). +class PYBIND11_EXPORT_EXCEPTION error_already_set : public std::exception { +public: + /// Fetches the current Python exception (using PyErr_Fetch()), which will clear the + /// current Python error indicator. + error_already_set() + : m_fetched_error{new detail::error_fetch_and_normalize("pybind11::error_already_set"), + m_fetched_error_deleter} {} + + /// The what() result is built lazily on demand. + /// WARNING: This member function needs to acquire the Python GIL. This can lead to + /// crashes (undefined behavior) if the Python interpreter is finalizing. + const char *what() const noexcept override; + + /// Restores the currently-held Python error (which will clear the Python error indicator first + /// if already set). + /// NOTE: This member function will always restore the normalized exception, which may or may + /// not be the original Python exception. + /// WARNING: The GIL must be held when this member function is called! + void restore() { m_fetched_error->restore(); } + + /// If it is impossible to raise the currently-held error, such as in a destructor, we can + /// write it out using Python's unraisable hook (`sys.unraisablehook`). The error context + /// should be some object whose `repr()` helps identify the location of the error. Python + /// already knows the type and value of the error, so there is no need to repeat that. void discard_as_unraisable(object err_context) { restore(); PyErr_WriteUnraisable(err_context.ptr()); } + /// An alternate version of `discard_as_unraisable()`, where a string provides information on + /// the location of the error. For example, `__func__` could be helpful. + /// WARNING: The GIL must be held when this member function is called! void discard_as_unraisable(const char *err_context) { discard_as_unraisable(reinterpret_steal<object>(PYBIND11_FROM_STRING(err_context))); } @@ -360,17 +756,57 @@ public: /// Check if the currently trapped error type matches the given Python exception class (or a /// subclass thereof). May also be passed a tuple to search for any exception class matches in /// the given tuple. - bool matches(handle exc) const { return PyErr_GivenExceptionMatches(m_type.ptr(), exc.ptr()); } + bool matches(handle exc) const { return m_fetched_error->matches(exc); } - const object& type() const { return m_type; } - const object& value() const { return m_value; } - const object& trace() const { return m_trace; } + const object &type() const { return m_fetched_error->m_type; } + const object &value() const { return m_fetched_error->m_value; } + const object &trace() const { return m_fetched_error->m_trace; } private: - object m_type, m_value, m_trace; + std::shared_ptr<detail::error_fetch_and_normalize> m_fetched_error; + + /// WARNING: This custom deleter needs to acquire the Python GIL. This can lead to + /// crashes (undefined behavior) if the Python interpreter is finalizing. + static void m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr); }; -/** \defgroup python_builtins _ +/// Replaces the current Python error indicator with the chosen error, performing a +/// 'raise from' to indicate that the chosen error was caused by the original error. +inline void raise_from(PyObject *type, const char *message) { + // Based on _PyErr_FormatVFromCause: + // https://github.com/python/cpython/blob/467ab194fc6189d9f7310c89937c51abeac56839/Python/errors.c#L405 + // See https://github.com/pybind/pybind11/pull/2112 for details. + PyObject *exc = nullptr, *val = nullptr, *val2 = nullptr, *tb = nullptr; + + assert(PyErr_Occurred()); + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != nullptr) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + assert(!PyErr_Occurred()); + + PyErr_SetString(type, message); + + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + Py_INCREF(val); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + PyErr_Restore(exc, val2, tb); +} + +/// Sets the current Python error indicator with the chosen error, performing a 'raise from' +/// from the error contained in error_already_set to indicate that the chosen error was +/// caused by the original error. +inline void raise_from(error_already_set &err, PyObject *type, const char *message) { + err.restore(); + raise_from(type, message); +} + +/** \defgroup python_builtins const_name Unless stated otherwise, the following C++ functions behave the same as their Python counterparts. */ @@ -381,20 +817,29 @@ private: `object` or a class which was exposed to Python as ``py::class_<T>``. \endrst */ template <typename T, detail::enable_if_t<std::is_base_of<object, T>::value, int> = 0> -bool isinstance(handle obj) { return T::check_(obj); } +bool isinstance(handle obj) { + return T::check_(obj); +} template <typename T, detail::enable_if_t<!std::is_base_of<object, T>::value, int> = 0> -bool isinstance(handle obj) { return detail::isinstance_generic(obj, typeid(T)); } +bool isinstance(handle obj) { + return detail::isinstance_generic(obj, typeid(T)); +} -template <> inline bool isinstance<handle>(handle) = delete; -template <> inline bool isinstance<object>(handle obj) { return obj.ptr() != nullptr; } +template <> +inline bool isinstance<handle>(handle) = delete; +template <> +inline bool isinstance<object>(handle obj) { + return obj.ptr() != nullptr; +} /// \ingroup python_builtins /// Return true if ``obj`` is an instance of the ``type``. inline bool isinstance(handle obj, handle type) { const auto result = PyObject_IsInstance(obj.ptr(), type.ptr()); - if (result == -1) + if (result == -1) { throw error_already_set(); + } return result != 0; } @@ -409,54 +854,66 @@ inline bool hasattr(handle obj, const char *name) { } inline void delattr(handle obj, handle name) { - if (PyObject_DelAttr(obj.ptr(), name.ptr()) != 0) { throw error_already_set(); } + if (PyObject_DelAttr(obj.ptr(), name.ptr()) != 0) { + throw error_already_set(); + } } inline void delattr(handle obj, const char *name) { - if (PyObject_DelAttrString(obj.ptr(), name) != 0) { throw error_already_set(); } + if (PyObject_DelAttrString(obj.ptr(), name) != 0) { + throw error_already_set(); + } } inline object getattr(handle obj, handle name) { PyObject *result = PyObject_GetAttr(obj.ptr(), name.ptr()); - if (!result) { throw error_already_set(); } + if (!result) { + throw error_already_set(); + } return reinterpret_steal<object>(result); } inline object getattr(handle obj, const char *name) { PyObject *result = PyObject_GetAttrString(obj.ptr(), name); - if (!result) { throw error_already_set(); } + if (!result) { + throw error_already_set(); + } return reinterpret_steal<object>(result); } inline object getattr(handle obj, handle name, handle default_) { if (PyObject *result = PyObject_GetAttr(obj.ptr(), name.ptr())) { return reinterpret_steal<object>(result); - } else { - PyErr_Clear(); - return reinterpret_borrow<object>(default_); } + PyErr_Clear(); + return reinterpret_borrow<object>(default_); } inline object getattr(handle obj, const char *name, handle default_) { if (PyObject *result = PyObject_GetAttrString(obj.ptr(), name)) { return reinterpret_steal<object>(result); - } else { - PyErr_Clear(); - return reinterpret_borrow<object>(default_); } + PyErr_Clear(); + return reinterpret_borrow<object>(default_); } inline void setattr(handle obj, handle name, handle value) { - if (PyObject_SetAttr(obj.ptr(), name.ptr(), value.ptr()) != 0) { throw error_already_set(); } + if (PyObject_SetAttr(obj.ptr(), name.ptr(), value.ptr()) != 0) { + throw error_already_set(); + } } inline void setattr(handle obj, const char *name, handle value) { - if (PyObject_SetAttrString(obj.ptr(), name, value.ptr()) != 0) { throw error_already_set(); } + if (PyObject_SetAttrString(obj.ptr(), name, value.ptr()) != 0) { + throw error_already_set(); + } } inline ssize_t hash(handle obj) { auto h = PyObject_Hash(obj.ptr()); - if (h == -1) { throw error_already_set(); } + if (h == -1) { + throw error_already_set(); + } return h; } @@ -465,68 +922,112 @@ inline ssize_t hash(handle obj) { PYBIND11_NAMESPACE_BEGIN(detail) inline handle get_function(handle value) { if (value) { -#if PY_MAJOR_VERSION >= 3 - if (PyInstanceMethod_Check(value.ptr())) + if (PyInstanceMethod_Check(value.ptr())) { value = PyInstanceMethod_GET_FUNCTION(value.ptr()); - else -#endif - if (PyMethod_Check(value.ptr())) + } else if (PyMethod_Check(value.ptr())) { value = PyMethod_GET_FUNCTION(value.ptr()); + } } return value; } -// Helper aliases/functions to support implicit casting of values given to python accessors/methods. -// When given a pyobject, this simply returns the pyobject as-is; for other C++ type, the value goes -// through pybind11::cast(obj) to convert it to an `object`. +// Reimplementation of python's dict helper functions to ensure that exceptions +// aren't swallowed (see #2862) + +// copied from cpython _PyDict_GetItemStringWithError +inline PyObject *dict_getitemstring(PyObject *v, const char *key) { + PyObject *kv = nullptr, *rv = nullptr; + kv = PyUnicode_FromString(key); + if (kv == nullptr) { + throw error_already_set(); + } + + rv = PyDict_GetItemWithError(v, kv); + Py_DECREF(kv); + if (rv == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + return rv; +} + +inline PyObject *dict_getitem(PyObject *v, PyObject *key) { + PyObject *rv = PyDict_GetItemWithError(v, key); + if (rv == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + return rv; +} + +// Helper aliases/functions to support implicit casting of values given to python +// accessors/methods. When given a pyobject, this simply returns the pyobject as-is; for other C++ +// type, the value goes through pybind11::cast(obj) to convert it to an `object`. template <typename T, enable_if_t<is_pyobject<T>::value, int> = 0> -auto object_or_cast(T &&o) -> decltype(std::forward<T>(o)) { return std::forward<T>(o); } +auto object_or_cast(T &&o) -> decltype(std::forward<T>(o)) { + return std::forward<T>(o); +} // The following casting version is implemented in cast.h: template <typename T, enable_if_t<!is_pyobject<T>::value, int> = 0> object object_or_cast(T &&o); // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4522) // warning C4522: multiple assignment operators specified template <typename Policy> class accessor : public object_api<accessor<Policy>> { using key_type = typename Policy::key_type; public: - accessor(handle obj, key_type key) : obj(obj), key(std::move(key)) { } + accessor(handle obj, key_type key) : obj(obj), key(std::move(key)) {} accessor(const accessor &) = default; - accessor(accessor &&) = default; + accessor(accessor &&) noexcept = default; - // accessor overload required to override default assignment operator (templates are not allowed - // to replace default compiler-generated assignments). + // accessor overload required to override default assignment operator (templates are not + // allowed to replace default compiler-generated assignments). void operator=(const accessor &a) && { std::move(*this).operator=(handle(a)); } void operator=(const accessor &a) & { operator=(handle(a)); } - template <typename T> void operator=(T &&value) && { + template <typename T> + void operator=(T &&value) && { Policy::set(obj, key, object_or_cast(std::forward<T>(value))); } - template <typename T> void operator=(T &&value) & { - get_cache() = reinterpret_borrow<object>(object_or_cast(std::forward<T>(value))); + template <typename T> + void operator=(T &&value) & { + get_cache() = ensure_object(object_or_cast(std::forward<T>(value))); } template <typename T = Policy> - PYBIND11_DEPRECATED("Use of obj.attr(...) as bool is deprecated in favor of pybind11::hasattr(obj, ...)") - explicit operator enable_if_t<std::is_same<T, accessor_policies::str_attr>::value || - std::is_same<T, accessor_policies::obj_attr>::value, bool>() const { + PYBIND11_DEPRECATED( + "Use of obj.attr(...) as bool is deprecated in favor of pybind11::hasattr(obj, ...)") + explicit + operator enable_if_t<std::is_same<T, accessor_policies::str_attr>::value + || std::is_same<T, accessor_policies::obj_attr>::value, + bool>() const { return hasattr(obj, key); } template <typename T = Policy> PYBIND11_DEPRECATED("Use of obj[key] as bool is deprecated in favor of obj.contains(key)") - explicit operator enable_if_t<std::is_same<T, accessor_policies::generic_item>::value, bool>() const { + explicit + operator enable_if_t<std::is_same<T, accessor_policies::generic_item>::value, bool>() const { return obj.contains(key); } + // NOLINTNEXTLINE(google-explicit-constructor) operator object() const { return get_cache(); } PyObject *ptr() const { return get_cache().ptr(); } - template <typename T> T cast() const { return get_cache().template cast<T>(); } + template <typename T> + T cast() const { + return get_cache().template cast<T>(); + } private: + static object ensure_object(object &&o) { return std::move(o); } + static object ensure_object(handle h) { return reinterpret_borrow<object>(h); } + object &get_cache() const { - if (!cache) { cache = Policy::get(obj, key); } + if (!cache) { + cache = Policy::get(obj, key); + } return cache; } @@ -535,6 +1036,7 @@ private: key_type key; mutable object cache; }; +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(accessor_policies) struct obj_attr { @@ -554,27 +1056,35 @@ struct generic_item { static object get(handle obj, handle key) { PyObject *result = PyObject_GetItem(obj.ptr(), key.ptr()); - if (!result) { throw error_already_set(); } + if (!result) { + throw error_already_set(); + } return reinterpret_steal<object>(result); } static void set(handle obj, handle key, handle val) { - if (PyObject_SetItem(obj.ptr(), key.ptr(), val.ptr()) != 0) { throw error_already_set(); } + if (PyObject_SetItem(obj.ptr(), key.ptr(), val.ptr()) != 0) { + throw error_already_set(); + } } }; struct sequence_item { using key_type = size_t; - static object get(handle obj, size_t index) { - PyObject *result = PySequence_GetItem(obj.ptr(), static_cast<ssize_t>(index)); - if (!result) { throw error_already_set(); } + template <typename IdxType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + static object get(handle obj, const IdxType &index) { + PyObject *result = PySequence_GetItem(obj.ptr(), ssize_t_cast(index)); + if (!result) { + throw error_already_set(); + } return reinterpret_steal<object>(result); } - static void set(handle obj, size_t index, handle val) { + template <typename IdxType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + static void set(handle obj, const IdxType &index, handle val) { // PySequence_SetItem does not steal a reference to 'val' - if (PySequence_SetItem(obj.ptr(), static_cast<ssize_t>(index), val.ptr()) != 0) { + if (PySequence_SetItem(obj.ptr(), ssize_t_cast(index), val.ptr()) != 0) { throw error_already_set(); } } @@ -583,15 +1093,19 @@ struct sequence_item { struct list_item { using key_type = size_t; - static object get(handle obj, size_t index) { - PyObject *result = PyList_GetItem(obj.ptr(), static_cast<ssize_t>(index)); - if (!result) { throw error_already_set(); } + template <typename IdxType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + static object get(handle obj, const IdxType &index) { + PyObject *result = PyList_GetItem(obj.ptr(), ssize_t_cast(index)); + if (!result) { + throw error_already_set(); + } return reinterpret_borrow<object>(result); } - static void set(handle obj, size_t index, handle val) { + template <typename IdxType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + static void set(handle obj, const IdxType &index, handle val) { // PyList_SetItem steals a reference to 'val' - if (PyList_SetItem(obj.ptr(), static_cast<ssize_t>(index), val.inc_ref().ptr()) != 0) { + if (PyList_SetItem(obj.ptr(), ssize_t_cast(index), val.inc_ref().ptr()) != 0) { throw error_already_set(); } } @@ -600,15 +1114,19 @@ struct list_item { struct tuple_item { using key_type = size_t; - static object get(handle obj, size_t index) { - PyObject *result = PyTuple_GetItem(obj.ptr(), static_cast<ssize_t>(index)); - if (!result) { throw error_already_set(); } + template <typename IdxType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + static object get(handle obj, const IdxType &index) { + PyObject *result = PyTuple_GetItem(obj.ptr(), ssize_t_cast(index)); + if (!result) { + throw error_already_set(); + } return reinterpret_borrow<object>(result); } - static void set(handle obj, size_t index, handle val) { + template <typename IdxType, detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + static void set(handle obj, const IdxType &index, handle val) { // PyTuple_SetItem steals a reference to 'val' - if (PyTuple_SetItem(obj.ptr(), static_cast<ssize_t>(index), val.inc_ref().ptr()) != 0) { + if (PyTuple_SetItem(obj.ptr(), ssize_t_cast(index), val.inc_ref().ptr()) != 0) { throw error_already_set(); } } @@ -628,28 +1146,56 @@ public: using pointer = typename Policy::pointer; generic_iterator() = default; - generic_iterator(handle seq, ssize_t index) : Policy(seq, index) { } + generic_iterator(handle seq, ssize_t index) : Policy(seq, index) {} + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference operator*() const { return Policy::dereference(); } + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference operator[](difference_type n) const { return *(*this + n); } pointer operator->() const { return **this; } - It &operator++() { Policy::increment(); return *this; } - It operator++(int) { auto copy = *this; Policy::increment(); return copy; } - It &operator--() { Policy::decrement(); return *this; } - It operator--(int) { auto copy = *this; Policy::decrement(); return copy; } - It &operator+=(difference_type n) { Policy::advance(n); return *this; } - It &operator-=(difference_type n) { Policy::advance(-n); return *this; } + It &operator++() { + Policy::increment(); + return *this; + } + It operator++(int) { + auto copy = *this; + Policy::increment(); + return copy; + } + It &operator--() { + Policy::decrement(); + return *this; + } + It operator--(int) { + auto copy = *this; + Policy::decrement(); + return copy; + } + It &operator+=(difference_type n) { + Policy::advance(n); + return *this; + } + It &operator-=(difference_type n) { + Policy::advance(-n); + return *this; + } - friend It operator+(const It &a, difference_type n) { auto copy = a; return copy += n; } + friend It operator+(const It &a, difference_type n) { + auto copy = a; + return copy += n; + } friend It operator+(difference_type n, const It &b) { return b + n; } - friend It operator-(const It &a, difference_type n) { auto copy = a; return copy -= n; } + friend It operator-(const It &a, difference_type n) { + auto copy = a; + return copy -= n; + } friend difference_type operator-(const It &a, const It &b) { return a.distance_to(b); } friend bool operator==(const It &a, const It &b) { return a.equal(b); } friend bool operator!=(const It &a, const It &b) { return !(a == b); } - friend bool operator< (const It &a, const It &b) { return b - a > 0; } - friend bool operator> (const It &a, const It &b) { return b < a; } + friend bool operator<(const It &a, const It &b) { return b - a > 0; } + friend bool operator>(const It &a, const It &b) { return b < a; } friend bool operator>=(const It &a, const It &b) { return !(a < b); } friend bool operator<=(const It &a, const It &b) { return !(a > b); } }; @@ -660,7 +1206,8 @@ template <typename T> struct arrow_proxy { T value; - arrow_proxy(T &&value) : value(std::move(value)) { } + // NOLINTNEXTLINE(google-explicit-constructor) + arrow_proxy(T &&value) noexcept : value(std::move(value)) {} T *operator->() const { return &value; } }; @@ -669,11 +1216,12 @@ class sequence_fast_readonly { protected: using iterator_category = std::random_access_iterator_tag; using value_type = handle; - using reference = const handle; + using reference = const handle; // PR #3263 using pointer = arrow_proxy<const handle>; - sequence_fast_readonly(handle obj, ssize_t n) : ptr(PySequence_Fast_ITEMS(obj.ptr()) + n) { } + sequence_fast_readonly(handle obj, ssize_t n) : ptr(PySequence_Fast_ITEMS(obj.ptr()) + n) {} + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference dereference() const { return *ptr; } void increment() { ++ptr; } void decrement() { --ptr; } @@ -693,7 +1241,7 @@ protected: using reference = sequence_accessor; using pointer = arrow_proxy<const sequence_accessor>; - sequence_slow_readwrite(handle obj, ssize_t index) : obj(obj), index(index) { } + sequence_slow_readwrite(handle obj, ssize_t index) : obj(obj), index(index) {} reference dereference() const { return {obj, static_cast<size_t>(index)}; } void increment() { ++index; } @@ -712,14 +1260,19 @@ class dict_readonly { protected: using iterator_category = std::forward_iterator_tag; using value_type = std::pair<handle, handle>; - using reference = const value_type; + using reference = const value_type; // PR #3263 using pointer = arrow_proxy<const value_type>; dict_readonly() = default; dict_readonly(handle obj, ssize_t pos) : obj(obj), pos(pos) { increment(); } + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference dereference() const { return {key, value}; } - void increment() { if (!PyDict_Next(obj.ptr(), &pos, &key, &value)) { pos = -1; } } + void increment() { + if (PyDict_Next(obj.ptr(), &pos, &key, &value) == 0) { + pos = -1; + } + } bool equal(const dict_readonly &b) const { return pos == b.pos; } private: @@ -745,38 +1298,47 @@ inline bool PyIterable_Check(PyObject *obj) { if (iter) { Py_DECREF(iter); return true; - } else { - PyErr_Clear(); - return false; } + PyErr_Clear(); + return false; } inline bool PyNone_Check(PyObject *o) { return o == Py_None; } inline bool PyEllipsis_Check(PyObject *o) { return o == Py_Ellipsis; } -inline bool PyUnicode_Check_Permissive(PyObject *o) { return PyUnicode_Check(o) || PYBIND11_BYTES_CHECK(o); } +#ifdef PYBIND11_STR_LEGACY_PERMISSIVE +inline bool PyUnicode_Check_Permissive(PyObject *o) { + return PyUnicode_Check(o) || PYBIND11_BYTES_CHECK(o); +} +# define PYBIND11_STR_CHECK_FUN detail::PyUnicode_Check_Permissive +#else +# define PYBIND11_STR_CHECK_FUN PyUnicode_Check +#endif inline bool PyStaticMethod_Check(PyObject *o) { return o->ob_type == &PyStaticMethod_Type; } class kwargs_proxy : public handle { public: - explicit kwargs_proxy(handle h) : handle(h) { } + explicit kwargs_proxy(handle h) : handle(h) {} }; class args_proxy : public handle { public: - explicit args_proxy(handle h) : handle(h) { } + explicit args_proxy(handle h) : handle(h) {} kwargs_proxy operator*() const { return kwargs_proxy(*this); } }; /// Python argument categories (using PEP 448 terms) -template <typename T> using is_keyword = std::is_base_of<arg, T>; -template <typename T> using is_s_unpacking = std::is_same<args_proxy, T>; // * unpacking -template <typename T> using is_ds_unpacking = std::is_same<kwargs_proxy, T>; // ** unpacking -template <typename T> using is_positional = satisfies_none_of<T, - is_keyword, is_s_unpacking, is_ds_unpacking ->; -template <typename T> using is_keyword_or_ds = satisfies_any_of<T, is_keyword, is_ds_unpacking>; +template <typename T> +using is_keyword = std::is_base_of<arg, T>; +template <typename T> +using is_s_unpacking = std::is_same<args_proxy, T>; // * unpacking +template <typename T> +using is_ds_unpacking = std::is_same<kwargs_proxy, T>; // ** unpacking +template <typename T> +using is_positional = satisfies_none_of<T, is_keyword, is_s_unpacking, is_ds_unpacking>; +template <typename T> +using is_keyword_or_ds = satisfies_any_of<T, is_keyword, is_ds_unpacking>; // Call argument collector forward declarations template <return_value_policy policy = return_value_policy::automatic_reference> @@ -790,44 +1352,60 @@ PYBIND11_NAMESPACE_END(detail) // inheriting ctors: `using Parent::Parent`. It's not an option right now because // the `using` statement triggers the parent deprecation warning even if the ctor // isn't even used. -#define PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ - public: \ - PYBIND11_DEPRECATED("Use reinterpret_borrow<"#Name">() or reinterpret_steal<"#Name">()") \ - Name(handle h, bool is_borrowed) : Parent(is_borrowed ? Parent(h, borrowed_t{}) : Parent(h, stolen_t{})) { } \ - Name(handle h, borrowed_t) : Parent(h, borrowed_t{}) { } \ - Name(handle h, stolen_t) : Parent(h, stolen_t{}) { } \ - PYBIND11_DEPRECATED("Use py::isinstance<py::python_type>(obj) instead") \ - bool check() const { return m_ptr != nullptr && (bool) CheckFun(m_ptr); } \ - static bool check_(handle h) { return h.ptr() != nullptr && CheckFun(h.ptr()); } \ - template <typename Policy_> \ - Name(const ::pybind11::detail::accessor<Policy_> &a) : Name(object(a)) { } - -#define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ - PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ - /* This is deliberately not 'explicit' to allow implicit conversion from object: */ \ - Name(const object &o) \ - : Parent(check_(o) ? o.inc_ref().ptr() : ConvertFun(o.ptr()), stolen_t{}) \ - { if (!m_ptr) throw error_already_set(); } \ - Name(object &&o) \ - : Parent(check_(o) ? o.release().ptr() : ConvertFun(o.ptr()), stolen_t{}) \ - { if (!m_ptr) throw error_already_set(); } - -#define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ - ::pybind11::type_error("Object of type '" + \ - ::pybind11::detail::get_fully_qualified_tp_name(Py_TYPE(o_ptr)) + \ - "' is not an instance of '" #Name "'") - -#define PYBIND11_OBJECT(Name, Parent, CheckFun) \ - PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ - /* This is deliberately not 'explicit' to allow implicit conversion from object: */ \ - Name(const object &o) : Parent(o) \ - { if (m_ptr && !check_(m_ptr)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); } \ - Name(object &&o) : Parent(std::move(o)) \ - { if (m_ptr && !check_(m_ptr)) throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); } - -#define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ - PYBIND11_OBJECT(Name, Parent, CheckFun) \ - Name() : Parent() { } +#define PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ +public: \ + PYBIND11_DEPRECATED("Use reinterpret_borrow<" #Name ">() or reinterpret_steal<" #Name ">()") \ + Name(handle h, bool is_borrowed) \ + : Parent(is_borrowed ? Parent(h, borrowed_t{}) : Parent(h, stolen_t{})) {} \ + Name(handle h, borrowed_t) : Parent(h, borrowed_t{}) {} \ + Name(handle h, stolen_t) : Parent(h, stolen_t{}) {} \ + PYBIND11_DEPRECATED("Use py::isinstance<py::python_type>(obj) instead") \ + bool check() const { return m_ptr != nullptr && (CheckFun(m_ptr) != 0); } \ + static bool check_(handle h) { return h.ptr() != nullptr && CheckFun(h.ptr()); } \ + template <typename Policy_> /* NOLINTNEXTLINE(google-explicit-constructor) */ \ + Name(const ::pybind11::detail::accessor<Policy_> &a) : Name(object(a)) {} + +#define PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ + PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ + /* This is deliberately not 'explicit' to allow implicit conversion from object: */ \ + /* NOLINTNEXTLINE(google-explicit-constructor) */ \ + Name(const object &o) \ + : Parent(check_(o) ? o.inc_ref().ptr() : ConvertFun(o.ptr()), stolen_t{}) { \ + if (!m_ptr) \ + throw ::pybind11::error_already_set(); \ + } \ + /* NOLINTNEXTLINE(google-explicit-constructor) */ \ + Name(object &&o) : Parent(check_(o) ? o.release().ptr() : ConvertFun(o.ptr()), stolen_t{}) { \ + if (!m_ptr) \ + throw ::pybind11::error_already_set(); \ + } + +#define PYBIND11_OBJECT_CVT_DEFAULT(Name, Parent, CheckFun, ConvertFun) \ + PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ + Name() = default; + +#define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ + ::pybind11::type_error("Object of type '" \ + + ::pybind11::detail::get_fully_qualified_tp_name(Py_TYPE(o_ptr)) \ + + "' is not an instance of '" #Name "'") + +#define PYBIND11_OBJECT(Name, Parent, CheckFun) \ + PYBIND11_OBJECT_COMMON(Name, Parent, CheckFun) \ + /* This is deliberately not 'explicit' to allow implicit conversion from object: */ \ + /* NOLINTNEXTLINE(google-explicit-constructor) */ \ + Name(const object &o) : Parent(o) { \ + if (m_ptr && !check_(m_ptr)) \ + throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); \ + } \ + /* NOLINTNEXTLINE(google-explicit-constructor) */ \ + Name(object &&o) : Parent(std::move(o)) { \ + if (m_ptr && !check_(m_ptr)) \ + throw PYBIND11_OBJECT_CHECK_FAILED(Name, m_ptr); \ + } + +#define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ + PYBIND11_OBJECT(Name, Parent, CheckFun) \ + Name() = default; /// \addtogroup pytypes /// @{ @@ -845,12 +1423,12 @@ public: using iterator_category = std::input_iterator_tag; using difference_type = ssize_t; using value_type = handle; - using reference = const handle; + using reference = const handle; // PR #3263 using pointer = const handle *; PYBIND11_OBJECT_DEFAULT(iterator, object, PyIter_Check) - iterator& operator++() { + iterator &operator++() { advance(); return *this; } @@ -861,15 +1439,19 @@ public: return rv; } + // NOLINTNEXTLINE(readability-const-return-type) // PR #3263 reference operator*() const { if (m_ptr && !value.ptr()) { - auto& self = const_cast<iterator &>(*this); + auto &self = const_cast<iterator &>(*this); self.advance(); } return value; } - pointer operator->() const { operator*(); return &value; } + pointer operator->() const { + operator*(); + return &value; + } /** \rst The value which marks the end of the iteration. ``it == iterator::sentinel()`` @@ -892,21 +1474,21 @@ public: private: void advance() { value = reinterpret_steal<object>(PyIter_Next(m_ptr)); - if (PyErr_Occurred()) { throw error_already_set(); } + if (value.ptr() == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } } private: object value = {}; }; - - class type : public object { public: PYBIND11_OBJECT(type, object, PyType_Check) /// Return a type handle from a handle or an object - static handle handle_of(handle h) { return handle((PyObject*) Py_TYPE(h.ptr())); } + static handle handle_of(handle h) { return handle((PyObject *) Py_TYPE(h.ptr())); } /// Return a type object from a handle or an object static type of(handle h) { return type(type::handle_of(h), borrowed_t{}); } @@ -915,14 +1497,16 @@ public: /// Convert C++ type to handle if previously registered. Does not convert /// standard types, like int, float. etc. yet. /// See https://github.com/pybind/pybind11/issues/2486 - template<typename T> + template <typename T> static handle handle_of(); /// Convert C++ type to type if previously registered. Does not convert /// standard types, like int, float. etc. yet. /// See https://github.com/pybind/pybind11/issues/2486 - template<typename T> - static type of() {return type(type::handle_of<T>(), borrowed_t{}); } + template <typename T> + static type of() { + return type(type::handle_of<T>(), borrowed_t{}); + } }; class iterable : public object { @@ -934,20 +1518,47 @@ class bytes; class str : public object { public: - PYBIND11_OBJECT_CVT(str, object, detail::PyUnicode_Check_Permissive, raw_str) - - str(const char *c, size_t n) - : object(PyUnicode_FromStringAndSize(c, (ssize_t) n), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate string object!"); + PYBIND11_OBJECT_CVT(str, object, PYBIND11_STR_CHECK_FUN, raw_str) + + template <typename SzType, detail::enable_if_t<std::is_integral<SzType>::value, int> = 0> + str(const char *c, const SzType &n) + : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { + if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } + pybind11_fail("Could not allocate string object!"); + } } - // 'explicit' is explicitly omitted from the following constructors to allow implicit conversion to py::str from C++ string-like objects - str(const char *c = "") - : object(PyUnicode_FromString(c), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate string object!"); + // 'explicit' is explicitly omitted from the following constructors to allow implicit + // conversion to py::str from C++ string-like objects + // NOLINTNEXTLINE(google-explicit-constructor) + str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) { + if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } + pybind11_fail("Could not allocate string object!"); + } } - str(const std::string &s) : str(s.data(), s.size()) { } + // NOLINTNEXTLINE(google-explicit-constructor) + str(const std::string &s) : str(s.data(), s.size()) {} + +#ifdef PYBIND11_HAS_STRING_VIEW + // enable_if is needed to avoid "ambiguous conversion" errors (see PR #3521). + template <typename T, detail::enable_if_t<std::is_same<T, std::string_view>::value, int> = 0> + // NOLINTNEXTLINE(google-explicit-constructor) + str(T s) : str(s.data(), s.size()) {} + +# ifdef PYBIND11_HAS_U8STRING + // reinterpret_cast here is safe (C++20 guarantees char8_t has the same size/alignment as char) + // NOLINTNEXTLINE(google-explicit-constructor) + str(std::u8string_view s) : str(reinterpret_cast<const char *>(s.data()), s.size()) {} +# endif + +#endif explicit str(const bytes &b); @@ -955,19 +1566,26 @@ public: Return a string representation of the object. This is analogous to the ``str()`` function in Python. \endrst */ - explicit str(handle h) : object(raw_str(h.ptr()), stolen_t{}) { if (!m_ptr) throw error_already_set(); } + explicit str(handle h) : object(raw_str(h.ptr()), stolen_t{}) { + if (!m_ptr) { + throw error_already_set(); + } + } + // NOLINTNEXTLINE(google-explicit-constructor) operator std::string() const { object temp = *this; if (PyUnicode_Check(m_ptr)) { temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(m_ptr)); - if (!temp) - pybind11_fail("Unable to extract string contents! (encoding issue)"); + if (!temp) { + throw error_already_set(); + } + } + char *buffer = nullptr; + ssize_t length = 0; + if (PyBytes_AsStringAndSize(temp.ptr(), &buffer, &length) != 0) { + throw error_already_set(); } - char *buffer; - ssize_t length; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) - pybind11_fail("Unable to extract string contents! (invalid type)"); return std::string(buffer, (size_t) length); } @@ -980,11 +1598,6 @@ private: /// Return string representation -- always returns a new reference, even if already a str static PyObject *raw_str(PyObject *op) { PyObject *str_value = PyObject_Str(op); -#if PY_MAJOR_VERSION < 3 - if (!str_value) throw error_already_set(); - PyObject *unicode = PyUnicode_FromEncodedObject(str_value, "utf-8", nullptr); - Py_XDECREF(str_value); str_value = unicode; -#endif return str_value; } }; @@ -1004,27 +1617,51 @@ public: PYBIND11_OBJECT(bytes, object, PYBIND11_BYTES_CHECK) // Allow implicit conversion: - bytes(const char *c = "") - : object(PYBIND11_BYTES_FROM_STRING(c), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); + // NOLINTNEXTLINE(google-explicit-constructor) + bytes(const char *c = "") : object(PYBIND11_BYTES_FROM_STRING(c), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate bytes object!"); + } } - bytes(const char *c, size_t n) - : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate bytes object!"); + template <typename SzType, detail::enable_if_t<std::is_integral<SzType>::value, int> = 0> + bytes(const char *c, const SzType &n) + : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, ssize_t_cast(n)), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate bytes object!"); + } } // Allow implicit conversion: - bytes(const std::string &s) : bytes(s.data(), s.size()) { } + // NOLINTNEXTLINE(google-explicit-constructor) + bytes(const std::string &s) : bytes(s.data(), s.size()) {} explicit bytes(const pybind11::str &s); - operator std::string() const { - char *buffer; - ssize_t length; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length)) - pybind11_fail("Unable to extract bytes contents!"); - return std::string(buffer, (size_t) length); + // NOLINTNEXTLINE(google-explicit-constructor) + operator std::string() const { return string_op<std::string>(); } + +#ifdef PYBIND11_HAS_STRING_VIEW + // enable_if is needed to avoid "ambiguous conversion" errors (see PR #3521). + template <typename T, detail::enable_if_t<std::is_same<T, std::string_view>::value, int> = 0> + // NOLINTNEXTLINE(google-explicit-constructor) + bytes(T s) : bytes(s.data(), s.size()) {} + + // Obtain a string view that views the current `bytes` buffer value. Note that this is only + // valid so long as the `bytes` instance remains alive and so generally should not outlive the + // lifetime of the `bytes` instance. + // NOLINTNEXTLINE(google-explicit-constructor) + operator std::string_view() const { return string_op<std::string_view>(); } +#endif +private: + template <typename T> + T string_op() const { + char *buffer = nullptr; + ssize_t length = 0; + if (PyBytes_AsStringAndSize(m_ptr, &buffer, &length) != 0) { + throw error_already_set(); + } + return {buffer, static_cast<size_t>(length)}; } }; // Note: breathe >= 4.17.0 will fail to build docs if the below two constructors @@ -1035,58 +1672,100 @@ inline bytes::bytes(const pybind11::str &s) { object temp = s; if (PyUnicode_Check(s.ptr())) { temp = reinterpret_steal<object>(PyUnicode_AsUTF8String(s.ptr())); - if (!temp) - pybind11_fail("Unable to extract string contents! (encoding issue)"); + if (!temp) { + throw error_already_set(); + } + } + char *buffer = nullptr; + ssize_t length = 0; + if (PyBytes_AsStringAndSize(temp.ptr(), &buffer, &length) != 0) { + throw error_already_set(); } - char *buffer; - ssize_t length; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length)) - pybind11_fail("Unable to extract string contents! (invalid type)"); auto obj = reinterpret_steal<object>(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length)); - if (!obj) + if (!obj) { pybind11_fail("Could not allocate bytes object!"); + } m_ptr = obj.release().ptr(); } -inline str::str(const bytes& b) { - char *buffer; - ssize_t length; - if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length)) - pybind11_fail("Unable to extract bytes contents!"); - auto obj = reinterpret_steal<object>(PyUnicode_FromStringAndSize(buffer, (ssize_t) length)); - if (!obj) +inline str::str(const bytes &b) { + char *buffer = nullptr; + ssize_t length = 0; + if (PyBytes_AsStringAndSize(b.ptr(), &buffer, &length) != 0) { + throw error_already_set(); + } + auto obj = reinterpret_steal<object>(PyUnicode_FromStringAndSize(buffer, length)); + if (!obj) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); + } m_ptr = obj.release().ptr(); } /// \addtogroup pytypes /// @{ +class bytearray : public object { +public: + PYBIND11_OBJECT_CVT(bytearray, object, PyByteArray_Check, PyByteArray_FromObject) + + template <typename SzType, detail::enable_if_t<std::is_integral<SzType>::value, int> = 0> + bytearray(const char *c, const SzType &n) + : object(PyByteArray_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate bytearray object!"); + } + } + + bytearray() : bytearray("", 0) {} + + explicit bytearray(const std::string &s) : bytearray(s.data(), s.size()) {} + + size_t size() const { return static_cast<size_t>(PyByteArray_Size(m_ptr)); } + + explicit operator std::string() const { + char *buffer = PyByteArray_AS_STRING(m_ptr); + ssize_t size = PyByteArray_GET_SIZE(m_ptr); + return std::string(buffer, static_cast<size_t>(size)); + } +}; +// Note: breathe >= 4.17.0 will fail to build docs if the below two constructors +// are included in the doxygen group; close here and reopen after as a workaround +/// @} pytypes + +/// \addtogroup pytypes +/// @{ class none : public object { public: PYBIND11_OBJECT(none, object, detail::PyNone_Check) - none() : object(Py_None, borrowed_t{}) { } + none() : object(Py_None, borrowed_t{}) {} }; class ellipsis : public object { public: PYBIND11_OBJECT(ellipsis, object, detail::PyEllipsis_Check) - ellipsis() : object(Py_Ellipsis, borrowed_t{}) { } + ellipsis() : object(Py_Ellipsis, borrowed_t{}) {} }; class bool_ : public object { public: PYBIND11_OBJECT_CVT(bool_, object, PyBool_Check, raw_bool) - bool_() : object(Py_False, borrowed_t{}) { } + bool_() : object(Py_False, borrowed_t{}) {} // Allow implicit conversion from and to `bool`: - bool_(bool value) : object(value ? Py_True : Py_False, borrowed_t{}) { } - operator bool() const { return m_ptr && PyLong_AsLong(m_ptr) != 0; } + // NOLINTNEXTLINE(google-explicit-constructor) + bool_(bool value) : object(value ? Py_True : Py_False, borrowed_t{}) {} + // NOLINTNEXTLINE(google-explicit-constructor) + operator bool() const { return (m_ptr != nullptr) && PyLong_AsLong(m_ptr) != 0; } private: /// Return the truth value of an object -- always returns a new reference static PyObject *raw_bool(PyObject *op) { const auto value = PyObject_IsTrue(op); - if (value == -1) return nullptr; - return handle(value ? Py_True : Py_False).inc_ref().ptr(); + if (value == -1) { + return nullptr; + } + return handle(value != 0 ? Py_True : Py_False).inc_ref().ptr(); } }; @@ -1097,51 +1776,47 @@ PYBIND11_NAMESPACE_BEGIN(detail) // unsigned type: (A)-1 != (B)-1 when A and B are unsigned types of different sizes). template <typename Unsigned> Unsigned as_unsigned(PyObject *o) { - if (sizeof(Unsigned) <= sizeof(unsigned long) -#if PY_VERSION_HEX < 0x03000000 - || PyInt_Check(o) -#endif - ) { + if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; } - else { - unsigned long long v = PyLong_AsUnsignedLongLong(o); - return v == (unsigned long long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; - } + unsigned long long v = PyLong_AsUnsignedLongLong(o); + return v == (unsigned long long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; } PYBIND11_NAMESPACE_END(detail) class int_ : public object { public: PYBIND11_OBJECT_CVT(int_, object, PYBIND11_LONG_CHECK, PyNumber_Long) - int_() : object(PyLong_FromLong(0), stolen_t{}) { } + int_() : object(PyLong_FromLong(0), stolen_t{}) {} // Allow implicit conversion from C++ integral types: - template <typename T, - detail::enable_if_t<std::is_integral<T>::value, int> = 0> + template <typename T, detail::enable_if_t<std::is_integral<T>::value, int> = 0> + // NOLINTNEXTLINE(google-explicit-constructor) int_(T value) { if (sizeof(T) <= sizeof(long)) { - if (std::is_signed<T>::value) + if (std::is_signed<T>::value) { m_ptr = PyLong_FromLong((long) value); - else + } else { m_ptr = PyLong_FromUnsignedLong((unsigned long) value); + } } else { - if (std::is_signed<T>::value) + if (std::is_signed<T>::value) { m_ptr = PyLong_FromLongLong((long long) value); - else + } else { m_ptr = PyLong_FromUnsignedLongLong((unsigned long long) value); + } + } + if (!m_ptr) { + pybind11_fail("Could not allocate int object!"); } - if (!m_ptr) pybind11_fail("Could not allocate int object!"); } - template <typename T, - detail::enable_if_t<std::is_integral<T>::value, int> = 0> + template <typename T, detail::enable_if_t<std::is_integral<T>::value, int> = 0> + // NOLINTNEXTLINE(google-explicit-constructor) operator T() const { - return std::is_unsigned<T>::value - ? detail::as_unsigned<T>(m_ptr) - : sizeof(T) <= sizeof(long) - ? (T) PyLong_AsLong(m_ptr) - : (T) PYBIND11_LONG_AS_LONGLONG(m_ptr); + return std::is_unsigned<T>::value ? detail::as_unsigned<T>(m_ptr) + : sizeof(T) <= sizeof(long) ? (T) PyLong_AsLong(m_ptr) + : (T) PYBIND11_LONG_AS_LONGLONG(m_ptr); } }; @@ -1149,46 +1824,80 @@ class float_ : public object { public: PYBIND11_OBJECT_CVT(float_, object, PyFloat_Check, PyNumber_Float) // Allow implicit conversion from float/double: + // NOLINTNEXTLINE(google-explicit-constructor) float_(float value) : object(PyFloat_FromDouble((double) value), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate float object!"); + if (!m_ptr) { + pybind11_fail("Could not allocate float object!"); + } } + // NOLINTNEXTLINE(google-explicit-constructor) float_(double value = .0) : object(PyFloat_FromDouble((double) value), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate float object!"); + if (!m_ptr) { + pybind11_fail("Could not allocate float object!"); + } } + // NOLINTNEXTLINE(google-explicit-constructor) operator float() const { return (float) PyFloat_AsDouble(m_ptr); } + // NOLINTNEXTLINE(google-explicit-constructor) operator double() const { return (double) PyFloat_AsDouble(m_ptr); } }; class weakref : public object { public: - PYBIND11_OBJECT_DEFAULT(weakref, object, PyWeakref_Check) + PYBIND11_OBJECT_CVT_DEFAULT(weakref, object, PyWeakref_Check, raw_weakref) explicit weakref(handle obj, handle callback = {}) : object(PyWeakref_NewRef(obj.ptr(), callback.ptr()), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate weak reference!"); + if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } + pybind11_fail("Could not allocate weak reference!"); + } } + +private: + static PyObject *raw_weakref(PyObject *o) { return PyWeakref_NewRef(o, nullptr); } }; class slice : public object { public: PYBIND11_OBJECT_DEFAULT(slice, object, PySlice_Check) - slice(ssize_t start_, ssize_t stop_, ssize_t step_) { - int_ start(start_), stop(stop_), step(step_); - m_ptr = PySlice_New(start.ptr(), stop.ptr(), step.ptr()); - if (!m_ptr) pybind11_fail("Could not allocate slice object!"); + slice(handle start, handle stop, handle step) + : object(PySlice_New(start.ptr(), stop.ptr(), step.ptr()), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate slice object!"); + } } - bool compute(size_t length, size_t *start, size_t *stop, size_t *step, - size_t *slicelength) const { + +#ifdef PYBIND11_HAS_OPTIONAL + slice(std::optional<ssize_t> start, std::optional<ssize_t> stop, std::optional<ssize_t> step) + : slice(index_to_object(start), index_to_object(stop), index_to_object(step)) {} +#else + slice(ssize_t start_, ssize_t stop_, ssize_t step_) + : slice(int_(start_), int_(stop_), int_(step_)) {} +#endif + + bool + compute(size_t length, size_t *start, size_t *stop, size_t *step, size_t *slicelength) const { return PySlice_GetIndicesEx((PYBIND11_SLICE_OBJECT *) m_ptr, - (ssize_t) length, (ssize_t *) start, - (ssize_t *) stop, (ssize_t *) step, - (ssize_t *) slicelength) == 0; + (ssize_t) length, + (ssize_t *) start, + (ssize_t *) stop, + (ssize_t *) step, + (ssize_t *) slicelength) + == 0; } - bool compute(ssize_t length, ssize_t *start, ssize_t *stop, ssize_t *step, - ssize_t *slicelength) const { - return PySlice_GetIndicesEx((PYBIND11_SLICE_OBJECT *) m_ptr, - length, start, - stop, step, - slicelength) == 0; + bool compute( + ssize_t length, ssize_t *start, ssize_t *stop, ssize_t *step, ssize_t *slicelength) const { + return PySlice_GetIndicesEx( + (PYBIND11_SLICE_OBJECT *) m_ptr, length, start, stop, step, slicelength) + == 0; + } + +private: + template <typename T> + static object index_to_object(T index) { + return index ? object(int_(*index)) : object(none()); } }; @@ -1196,77 +1905,146 @@ class capsule : public object { public: PYBIND11_OBJECT_DEFAULT(capsule, object, PyCapsule_CheckExact) PYBIND11_DEPRECATED("Use reinterpret_borrow<capsule>() or reinterpret_steal<capsule>()") - capsule(PyObject *ptr, bool is_borrowed) : object(is_borrowed ? object(ptr, borrowed_t{}) : object(ptr, stolen_t{})) { } + capsule(PyObject *ptr, bool is_borrowed) + : object(is_borrowed ? object(ptr, borrowed_t{}) : object(ptr, stolen_t{})) {} - explicit capsule(const void *value, const char *name = nullptr, void (*destructor)(PyObject *) = nullptr) + explicit capsule(const void *value, + const char *name = nullptr, + PyCapsule_Destructor destructor = nullptr) : object(PyCapsule_New(const_cast<void *>(value), name, destructor), stolen_t{}) { - if (!m_ptr) - pybind11_fail("Could not allocate capsule object!"); + if (!m_ptr) { + throw error_already_set(); + } } - PYBIND11_DEPRECATED("Please pass a destructor that takes a void pointer as input") - capsule(const void *value, void (*destruct)(PyObject *)) - : object(PyCapsule_New(const_cast<void*>(value), nullptr, destruct), stolen_t{}) { - if (!m_ptr) - pybind11_fail("Could not allocate capsule object!"); + PYBIND11_DEPRECATED("Please use the ctor with value, name, destructor args") + capsule(const void *value, PyCapsule_Destructor destructor) + : object(PyCapsule_New(const_cast<void *>(value), nullptr, destructor), stolen_t{}) { + if (!m_ptr) { + throw error_already_set(); + } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast<void *>(value), nullptr, [](PyObject *o) { - auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o)); - void *ptr = PyCapsule_GetPointer(o, nullptr); - destructor(ptr); - }); - - if (!m_ptr) - pybind11_fail("Could not allocate capsule object!"); + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) - pybind11_fail("Could not set capsule context!"); + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } - capsule(void (*destructor)()) { + explicit capsule(void (*destructor)()) { m_ptr = PyCapsule_New(reinterpret_cast<void *>(destructor), nullptr, [](PyObject *o) { - auto destructor = reinterpret_cast<void (*)()>(PyCapsule_GetPointer(o, nullptr)); + const char *name = get_name_in_error_scope(o); + auto destructor = reinterpret_cast<void (*)()>(PyCapsule_GetPointer(o, name)); + if (destructor == nullptr) { + throw error_already_set(); + } destructor(); }); - if (!m_ptr) - pybind11_fail("Could not allocate capsule object!"); + if (!m_ptr) { + throw error_already_set(); + } } - template <typename T> operator T *() const { + template <typename T> + operator T *() const { // NOLINT(google-explicit-constructor) return get_pointer<T>(); } /// Get the pointer the capsule holds. - template<typename T = void> - T* get_pointer() const { - auto name = this->name(); + template <typename T = void> + T *get_pointer() const { + const auto *name = this->name(); T *result = static_cast<T *>(PyCapsule_GetPointer(m_ptr, name)); - if (!result) pybind11_fail("Unable to extract capsule contents!"); + if (!result) { + throw error_already_set(); + } return result; } /// Replaces a capsule's pointer *without* calling the destructor on the existing one. void set_pointer(const void *value) { - if (PyCapsule_SetPointer(m_ptr, const_cast<void *>(value)) != 0) - pybind11_fail("Could not set capsule pointer"); + if (PyCapsule_SetPointer(m_ptr, const_cast<void *>(value)) != 0) { + throw error_already_set(); + } + } + + const char *name() const { + const char *name = PyCapsule_GetName(m_ptr); + if ((name == nullptr) && PyErr_Occurred()) { + throw error_already_set(); + } + return name; + } + + /// Replaces a capsule's name *without* calling the destructor on the existing one. + void set_name(const char *new_name) { + if (PyCapsule_SetName(m_ptr, new_name) != 0) { + throw error_already_set(); + } + } + +private: + static const char *get_name_in_error_scope(PyObject *o) { + error_scope error_guard; + + const char *name = PyCapsule_GetName(o); + if ((name == nullptr) && PyErr_Occurred()) { + // write out and consume error raised by call to PyCapsule_GetName + PyErr_WriteUnraisable(o); + } + + return name; } - const char *name() const { return PyCapsule_GetName(m_ptr); } + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast<void *>(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast<void (*)(void *)>(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast<void *>(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { public: PYBIND11_OBJECT_CVT(tuple, object, PyTuple_Check, PySequence_Tuple) - explicit tuple(size_t size = 0) : object(PyTuple_New((ssize_t) size), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate tuple object!"); + template <typename SzType = ssize_t, + detail::enable_if_t<std::is_integral<SzType>::value, int> = 0> + // Some compilers generate link errors when using `const SzType &` here: + explicit tuple(SzType size = 0) : object(PyTuple_New(ssize_t_cast(size)), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate tuple object!"); + } } size_t size() const { return (size_t) PyTuple_Size(m_ptr); } bool empty() const { return size() == 0; } detail::tuple_accessor operator[](size_t index) const { return {*this, index}; } - detail::item_accessor operator[](handle h) const { return object::operator[](h); } + template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> + detail::item_accessor operator[](T &&o) const { + return object::operator[](std::forward<T>(o)); + } detail::tuple_iterator begin() const { return {*this, 0}; } detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } }; @@ -1275,37 +2053,45 @@ public: // fails to compile enable_if_t<all_of<is_keyword_or_ds<Args>...>::value> part below // (tested with ICC 2021.1 Beta 20200827). template <typename... Args> -constexpr bool args_are_all_keyword_or_ds() -{ - return detail::all_of<detail::is_keyword_or_ds<Args>...>::value; +constexpr bool args_are_all_keyword_or_ds() { + return detail::all_of<detail::is_keyword_or_ds<Args>...>::value; } class dict : public object { public: PYBIND11_OBJECT_CVT(dict, object, PyDict_Check, raw_dict) dict() : object(PyDict_New(), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate dict object!"); + if (!m_ptr) { + pybind11_fail("Could not allocate dict object!"); + } } template <typename... Args, typename = detail::enable_if_t<args_are_all_keyword_or_ds<Args...>()>, - // MSVC workaround: it can't compile an out-of-line definition, so defer the collector + // MSVC workaround: it can't compile an out-of-line definition, so defer the + // collector typename collector = detail::deferred_t<detail::unpacking_collector<>, Args...>> - explicit dict(Args &&...args) : dict(collector(std::forward<Args>(args)...).kwargs()) { } + explicit dict(Args &&...args) : dict(collector(std::forward<Args>(args)...).kwargs()) {} size_t size() const { return (size_t) PyDict_Size(m_ptr); } bool empty() const { return size() == 0; } detail::dict_iterator begin() const { return {*this, 0}; } detail::dict_iterator end() const { return {}; } - void clear() const { PyDict_Clear(ptr()); } - template <typename T> bool contains(T &&key) const { - return PyDict_Contains(m_ptr, detail::object_or_cast(std::forward<T>(key)).ptr()) == 1; + void clear() /* py-non-const */ { PyDict_Clear(ptr()); } + template <typename T> + bool contains(T &&key) const { + auto result = PyDict_Contains(m_ptr, detail::object_or_cast(std::forward<T>(key)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } private: /// Call the `dict` Python type -- always returns a new reference static PyObject *raw_dict(PyObject *op) { - if (PyDict_Check(op)) + if (PyDict_Check(op)) { return handle(op).inc_ref().ptr(); + } return PyObject_CallFunctionObjArgs((PyObject *) &PyDict_Type, op, nullptr); } }; @@ -1315,13 +2101,17 @@ public: PYBIND11_OBJECT_DEFAULT(sequence, object, PySequence_Check) size_t size() const { ssize_t result = PySequence_Size(m_ptr); - if (result == -1) + if (result == -1) { throw error_already_set(); + } return (size_t) result; } bool empty() const { return size() == 0; } detail::sequence_accessor operator[](size_t index) const { return {*this, index}; } - detail::item_accessor operator[](handle h) const { return object::operator[](h); } + template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> + detail::item_accessor operator[](T &&o) const { + return object::operator[](std::forward<T>(o)); + } detail::sequence_iterator begin() const { return {*this, 0}; } detail::sequence_iterator end() const { return {*this, PySequence_Size(m_ptr)}; } }; @@ -1329,42 +2119,82 @@ public: class list : public object { public: PYBIND11_OBJECT_CVT(list, object, PyList_Check, PySequence_List) - explicit list(size_t size = 0) : object(PyList_New((ssize_t) size), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate list object!"); + template <typename SzType = ssize_t, + detail::enable_if_t<std::is_integral<SzType>::value, int> = 0> + // Some compilers generate link errors when using `const SzType &` here: + explicit list(SzType size = 0) : object(PyList_New(ssize_t_cast(size)), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate list object!"); + } } size_t size() const { return (size_t) PyList_Size(m_ptr); } bool empty() const { return size() == 0; } detail::list_accessor operator[](size_t index) const { return {*this, index}; } - detail::item_accessor operator[](handle h) const { return object::operator[](h); } + template <typename T, detail::enable_if_t<detail::is_pyobject<T>::value, int> = 0> + detail::item_accessor operator[](T &&o) const { + return object::operator[](std::forward<T>(o)); + } detail::list_iterator begin() const { return {*this, 0}; } detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } - template <typename T> void append(T &&val) const { - PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()); + template <typename T> + void append(T &&val) /* py-non-const */ { + if (PyList_Append(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) != 0) { + throw error_already_set(); + } } - template <typename T> void insert(size_t index, T &&val) const { - PyList_Insert(m_ptr, static_cast<ssize_t>(index), - detail::object_or_cast(std::forward<T>(val)).ptr()); + template <typename IdxType, + typename ValType, + detail::enable_if_t<std::is_integral<IdxType>::value, int> = 0> + void insert(const IdxType &index, ValType &&val) /* py-non-const */ { + if (PyList_Insert(m_ptr, + ssize_t_cast(index), + detail::object_or_cast(std::forward<ValType>(val)).ptr()) + != 0) { + throw error_already_set(); + } } }; -class args : public tuple { PYBIND11_OBJECT_DEFAULT(args, tuple, PyTuple_Check) }; -class kwargs : public dict { PYBIND11_OBJECT_DEFAULT(kwargs, dict, PyDict_Check) }; +class args : public tuple { + PYBIND11_OBJECT_DEFAULT(args, tuple, PyTuple_Check) +}; +class kwargs : public dict { + PYBIND11_OBJECT_DEFAULT(kwargs, dict, PyDict_Check) +}; -class set : public object { +class anyset : public object { public: - PYBIND11_OBJECT_CVT(set, object, PySet_Check, PySet_New) - set() : object(PySet_New(nullptr), stolen_t{}) { - if (!m_ptr) pybind11_fail("Could not allocate set object!"); - } - size_t size() const { return (size_t) PySet_Size(m_ptr); } + PYBIND11_OBJECT(anyset, object, PyAnySet_Check) + size_t size() const { return static_cast<size_t>(PySet_Size(m_ptr)); } bool empty() const { return size() == 0; } - template <typename T> bool add(T &&val) const { - return PySet_Add(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) == 0; + template <typename T> + bool contains(T &&val) const { + auto result = PySet_Contains(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } - void clear() const { PySet_Clear(m_ptr); } - template <typename T> bool contains(T &&val) const { - return PySet_Contains(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) == 1; +}; + +class set : public anyset { +public: + PYBIND11_OBJECT_CVT(set, anyset, PySet_Check, PySet_New) + set() : anyset(PySet_New(nullptr), stolen_t{}) { + if (!m_ptr) { + pybind11_fail("Could not allocate set object!"); + } } + template <typename T> + bool add(T &&val) /* py-non-const */ { + return PySet_Add(m_ptr, detail::object_or_cast(std::forward<T>(val)).ptr()) == 0; + } + void clear() /* py-non-const */ { PySet_Clear(m_ptr); } +}; + +class frozenset : public anyset { +public: + PYBIND11_OBJECT_CVT(frozenset, anyset, PyFrozenSet_Check, PyFrozenSet_New) }; class function : public object { @@ -1372,8 +2202,9 @@ public: PYBIND11_OBJECT_DEFAULT(function, object, PyCallable_Check) handle cpp_function() const { handle fun = detail::get_function(m_ptr); - if (fun && PyCFunction_Check(fun.ptr())) + if (fun && PyCFunction_Check(fun.ptr())) { return fun; + } return handle(); } bool is_cpp_function() const { return (bool) cpp_function(); } @@ -1390,7 +2221,9 @@ public: buffer_info request(bool writable = false) const { int flags = PyBUF_STRIDES | PyBUF_FORMAT; - if (writable) flags |= PyBUF_WRITABLE; + if (writable) { + flags |= PyBUF_WRITABLE; + } auto *view = new Py_buffer(); if (PyObject_GetBuffer(m_ptr, view, flags) != 0) { delete view; @@ -1413,15 +2246,16 @@ public: For creating a ``memoryview`` from objects that support buffer protocol, use ``memoryview(const object& obj)`` instead of this constructor. \endrst */ - explicit memoryview(const buffer_info& info) { - if (!info.view()) + explicit memoryview(const buffer_info &info) { + if (!info.view()) { pybind11_fail("Prohibited to create memoryview without Py_buffer"); + } // Note: PyMemoryView_FromBuffer never increments obj reference. - m_ptr = (info.view()->obj) ? - PyMemoryView_FromObject(info.view()->obj) : - PyMemoryView_FromBuffer(info.view()); - if (!m_ptr) + m_ptr = (info.view()->obj) ? PyMemoryView_FromObject(info.view()->obj) + : PyMemoryView_FromBuffer(info.view()); + if (!m_ptr) { pybind11_fail("Unable to create memoryview from buffer descriptor"); + } } /** \rst @@ -1434,7 +2268,8 @@ public: See also: Python C API documentation for `PyMemoryView_FromBuffer`_. - .. _PyMemoryView_FromBuffer: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromBuffer + .. _PyMemoryView_FromBuffer: + https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromBuffer :param ptr: Pointer to the buffer. :param itemsize: Byte size of an element. @@ -1447,37 +2282,43 @@ public: :param readonly: Flag to indicate if the underlying storage may be written to. \endrst */ - static memoryview from_buffer( - void *ptr, ssize_t itemsize, const char *format, - detail::any_container<ssize_t> shape, - detail::any_container<ssize_t> strides, bool readonly = false); - - static memoryview from_buffer( - const void *ptr, ssize_t itemsize, const char *format, - detail::any_container<ssize_t> shape, - detail::any_container<ssize_t> strides) { + static memoryview from_buffer(void *ptr, + ssize_t itemsize, + const char *format, + detail::any_container<ssize_t> shape, + detail::any_container<ssize_t> strides, + bool readonly = false); + + static memoryview from_buffer(const void *ptr, + ssize_t itemsize, + const char *format, + detail::any_container<ssize_t> shape, + detail::any_container<ssize_t> strides) { return memoryview::from_buffer( - const_cast<void*>(ptr), itemsize, format, shape, strides, true); - } - - template<typename T> - static memoryview from_buffer( - T *ptr, detail::any_container<ssize_t> shape, - detail::any_container<ssize_t> strides, bool readonly = false) { + const_cast<void *>(ptr), itemsize, format, std::move(shape), std::move(strides), true); + } + + template <typename T> + static memoryview from_buffer(T *ptr, + detail::any_container<ssize_t> shape, + detail::any_container<ssize_t> strides, + bool readonly = false) { + return memoryview::from_buffer(reinterpret_cast<void *>(ptr), + sizeof(T), + format_descriptor<T>::value, + std::move(shape), + std::move(strides), + readonly); + } + + template <typename T> + static memoryview from_buffer(const T *ptr, + detail::any_container<ssize_t> shape, + detail::any_container<ssize_t> strides) { return memoryview::from_buffer( - reinterpret_cast<void*>(ptr), sizeof(T), - format_descriptor<T>::value, shape, strides, readonly); + const_cast<T *>(ptr), std::move(shape), std::move(strides), true); } - template<typename T> - static memoryview from_buffer( - const T *ptr, detail::any_container<ssize_t> shape, - detail::any_container<ssize_t> strides) { - return memoryview::from_buffer( - const_cast<T*>(ptr), shape, strides, true); - } - -#if PY_MAJOR_VERSION >= 3 /** \rst Creates ``memoryview`` from static memory. @@ -1485,56 +2326,65 @@ public: managed by Python. The caller is responsible for managing the lifetime of ``mem``, which MUST outlive the memoryview constructed here. - This method is not available in Python 2. - See also: Python C API documentation for `PyMemoryView_FromBuffer`_. - .. _PyMemoryView_FromMemory: https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromMemory + .. _PyMemoryView_FromMemory: + https://docs.python.org/c-api/memoryview.html#c.PyMemoryView_FromMemory \endrst */ static memoryview from_memory(void *mem, ssize_t size, bool readonly = false) { - PyObject* ptr = PyMemoryView_FromMemory( - reinterpret_cast<char*>(mem), size, - (readonly) ? PyBUF_READ : PyBUF_WRITE); - if (!ptr) + PyObject *ptr = PyMemoryView_FromMemory( + reinterpret_cast<char *>(mem), size, (readonly) ? PyBUF_READ : PyBUF_WRITE); + if (!ptr) { pybind11_fail("Could not allocate memoryview object!"); + } return memoryview(object(ptr, stolen_t{})); } static memoryview from_memory(const void *mem, ssize_t size) { - return memoryview::from_memory(const_cast<void*>(mem), size, true); + return memoryview::from_memory(const_cast<void *>(mem), size, true); + } + +#ifdef PYBIND11_HAS_STRING_VIEW + static memoryview from_memory(std::string_view mem) { + return from_memory(const_cast<char *>(mem.data()), static_cast<ssize_t>(mem.size()), true); } #endif }; -#ifndef DOXYGEN_SHOULD_SKIP_THIS -inline memoryview memoryview::from_buffer( - void *ptr, ssize_t itemsize, const char* format, - detail::any_container<ssize_t> shape, - detail::any_container<ssize_t> strides, bool readonly) { +/// @cond DUPLICATE +inline memoryview memoryview::from_buffer(void *ptr, + ssize_t itemsize, + const char *format, + detail::any_container<ssize_t> shape, + detail::any_container<ssize_t> strides, + bool readonly) { size_t ndim = shape->size(); - if (ndim != strides->size()) + if (ndim != strides->size()) { pybind11_fail("memoryview: shape length doesn't match strides length"); - ssize_t size = ndim ? 1 : 0; - for (size_t i = 0; i < ndim; ++i) + } + ssize_t size = ndim != 0u ? 1 : 0; + for (size_t i = 0; i < ndim; ++i) { size *= (*shape)[i]; + } Py_buffer view; view.buf = ptr; view.obj = nullptr; view.len = size * itemsize; view.readonly = static_cast<int>(readonly); view.itemsize = itemsize; - view.format = const_cast<char*>(format); + view.format = const_cast<char *>(format); view.ndim = static_cast<int>(ndim); view.shape = shape->data(); view.strides = strides->data(); view.suboffsets = nullptr; view.internal = nullptr; - PyObject* obj = PyMemoryView_FromBuffer(&view); - if (!obj) + PyObject *obj = PyMemoryView_FromBuffer(&view); + if (!obj) { throw error_already_set(); + } return memoryview(object(obj, stolen_t{})); } -#endif // DOXYGEN_SHOULD_SKIP_THIS +/// @endcond /// @} pytypes /// \addtogroup python_builtins @@ -1543,19 +2393,16 @@ inline memoryview memoryview::from_buffer( /// Get the length of a Python object. inline size_t len(handle h) { ssize_t result = PyObject_Length(h.ptr()); - if (result < 0) + if (result < 0) { throw error_already_set(); + } return (size_t) result; } /// Get the length hint of a Python object. /// Returns 0 when this cannot be determined. inline size_t len_hint(handle h) { -#if PY_VERSION_HEX >= 0x03040000 ssize_t result = PyObject_LengthHint(h.ptr(), 0); -#else - ssize_t result = PyObject_Length(h.ptr()); -#endif if (result < 0) { // Sometimes a length can't be determined at all (eg generators) // In which case simply return 0 @@ -1567,102 +2414,139 @@ inline size_t len_hint(handle h) { inline str repr(handle h) { PyObject *str_value = PyObject_Repr(h.ptr()); - if (!str_value) throw error_already_set(); -#if PY_MAJOR_VERSION < 3 - PyObject *unicode = PyUnicode_FromEncodedObject(str_value, "utf-8", nullptr); - Py_XDECREF(str_value); str_value = unicode; - if (!str_value) throw error_already_set(); -#endif + if (!str_value) { + throw error_already_set(); + } return reinterpret_steal<str>(str_value); } inline iterator iter(handle obj) { PyObject *result = PyObject_GetIter(obj.ptr()); - if (!result) { throw error_already_set(); } + if (!result) { + throw error_already_set(); + } return reinterpret_steal<iterator>(result); } /// @} python_builtins PYBIND11_NAMESPACE_BEGIN(detail) -template <typename D> iterator object_api<D>::begin() const { return iter(derived()); } -template <typename D> iterator object_api<D>::end() const { return iterator::sentinel(); } -template <typename D> item_accessor object_api<D>::operator[](handle key) const { +template <typename D> +iterator object_api<D>::begin() const { + return iter(derived()); +} +template <typename D> +iterator object_api<D>::end() const { + return iterator::sentinel(); +} +template <typename D> +item_accessor object_api<D>::operator[](handle key) const { return {derived(), reinterpret_borrow<object>(key)}; } -template <typename D> item_accessor object_api<D>::operator[](const char *key) const { +template <typename D> +item_accessor object_api<D>::operator[](object &&key) const { + return {derived(), std::move(key)}; +} +template <typename D> +item_accessor object_api<D>::operator[](const char *key) const { return {derived(), pybind11::str(key)}; } -template <typename D> obj_attr_accessor object_api<D>::attr(handle key) const { +template <typename D> +obj_attr_accessor object_api<D>::attr(handle key) const { return {derived(), reinterpret_borrow<object>(key)}; } -template <typename D> str_attr_accessor object_api<D>::attr(const char *key) const { +template <typename D> +obj_attr_accessor object_api<D>::attr(object &&key) const { + return {derived(), std::move(key)}; +} +template <typename D> +str_attr_accessor object_api<D>::attr(const char *key) const { return {derived(), key}; } -template <typename D> args_proxy object_api<D>::operator*() const { +template <typename D> +args_proxy object_api<D>::operator*() const { return args_proxy(derived().ptr()); } -template <typename D> template <typename T> bool object_api<D>::contains(T &&item) const { +template <typename D> +template <typename T> +bool object_api<D>::contains(T &&item) const { return attr("__contains__")(std::forward<T>(item)).template cast<bool>(); } template <typename D> -pybind11::str object_api<D>::str() const { return pybind11::str(derived()); } +pybind11::str object_api<D>::str() const { + return pybind11::str(derived()); +} template <typename D> -str_attr_accessor object_api<D>::doc() const { return attr("__doc__"); } +str_attr_accessor object_api<D>::doc() const { + return attr("__doc__"); +} template <typename D> -handle object_api<D>::get_type() const { return type::handle_of(derived()); } +handle object_api<D>::get_type() const { + return type::handle_of(derived()); +} template <typename D> bool object_api<D>::rich_compare(object_api const &other, int value) const { int rv = PyObject_RichCompareBool(derived().ptr(), other.derived().ptr(), value); - if (rv == -1) + if (rv == -1) { throw error_already_set(); + } return rv == 1; } -#define PYBIND11_MATH_OPERATOR_UNARY(op, fn) \ - template <typename D> object object_api<D>::op() const { \ - object result = reinterpret_steal<object>(fn(derived().ptr())); \ - if (!result.ptr()) \ - throw error_already_set(); \ - return result; \ - } - -#define PYBIND11_MATH_OPERATOR_BINARY(op, fn) \ - template <typename D> \ - object object_api<D>::op(object_api const &other) const { \ - object result = reinterpret_steal<object>( \ - fn(derived().ptr(), other.derived().ptr())); \ - if (!result.ptr()) \ - throw error_already_set(); \ - return result; \ - } - -PYBIND11_MATH_OPERATOR_UNARY (operator~, PyNumber_Invert) -PYBIND11_MATH_OPERATOR_UNARY (operator-, PyNumber_Negative) -PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add) -PYBIND11_MATH_OPERATOR_BINARY(operator+=, PyNumber_InPlaceAdd) -PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract) -PYBIND11_MATH_OPERATOR_BINARY(operator-=, PyNumber_InPlaceSubtract) -PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply) -PYBIND11_MATH_OPERATOR_BINARY(operator*=, PyNumber_InPlaceMultiply) -PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide) -PYBIND11_MATH_OPERATOR_BINARY(operator/=, PyNumber_InPlaceTrueDivide) -PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or) -PYBIND11_MATH_OPERATOR_BINARY(operator|=, PyNumber_InPlaceOr) -PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And) -PYBIND11_MATH_OPERATOR_BINARY(operator&=, PyNumber_InPlaceAnd) -PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor) -PYBIND11_MATH_OPERATOR_BINARY(operator^=, PyNumber_InPlaceXor) -PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift) -PYBIND11_MATH_OPERATOR_BINARY(operator<<=, PyNumber_InPlaceLshift) -PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift) -PYBIND11_MATH_OPERATOR_BINARY(operator>>=, PyNumber_InPlaceRshift) +#define PYBIND11_MATH_OPERATOR_UNARY(op, fn) \ + template <typename D> \ + object object_api<D>::op() const { \ + object result = reinterpret_steal<object>(fn(derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + +#define PYBIND11_MATH_OPERATOR_BINARY(op, fn) \ + template <typename D> \ + object object_api<D>::op(object_api const &other) const { \ + object result = reinterpret_steal<object>(fn(derived().ptr(), other.derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + +#define PYBIND11_MATH_OPERATOR_BINARY_INPLACE(iop, fn) \ + template <typename D> \ + object object_api<D>::iop(object_api const &other) { \ + object result = reinterpret_steal<object>(fn(derived().ptr(), other.derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + +PYBIND11_MATH_OPERATOR_UNARY(operator~, PyNumber_Invert) +PYBIND11_MATH_OPERATOR_UNARY(operator-, PyNumber_Negative) +PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator+=, PyNumber_InPlaceAdd) +PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator-=, PyNumber_InPlaceSubtract) +PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator*=, PyNumber_InPlaceMultiply) +PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator/=, PyNumber_InPlaceTrueDivide) +PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator|=, PyNumber_InPlaceOr) +PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator&=, PyNumber_InPlaceAnd) +PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator^=, PyNumber_InPlaceXor) +PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator<<=, PyNumber_InPlaceLshift) +PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_UNARY #undef PYBIND11_MATH_OPERATOR_BINARY +#undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 721bb669..f39f44f7 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -10,42 +10,26 @@ #pragma once #include "pybind11.h" -#include <set> -#include <unordered_set> +#include "detail/common.h" + +#include <deque> +#include <list> #include <map> +#include <ostream> +#include <set> #include <unordered_map> -#include <iostream> -#include <list> -#include <deque> +#include <unordered_set> #include <valarray> -#if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable: 4127) // warning C4127: Conditional expression is constant -#endif - -#ifdef __has_include -// std::optional (but including it in c++14 mode isn't allowed) -# if defined(PYBIND11_CPP17) && __has_include(<optional>) +// See `detail/common.h` for implementation of these guards. +#if defined(PYBIND11_HAS_OPTIONAL) # include <optional> -# define PYBIND11_HAS_OPTIONAL 1 -# endif -// std::experimental::optional (but not allowed in c++11 mode) -# if defined(PYBIND11_CPP14) && (__has_include(<experimental/optional>) && \ - !__has_include(<optional>)) +#elif defined(PYBIND11_HAS_EXP_OPTIONAL) # include <experimental/optional> -# define PYBIND11_HAS_EXP_OPTIONAL 1 -# endif -// std::variant -# if defined(PYBIND11_CPP17) && __has_include(<variant>) +#endif + +#if defined(PYBIND11_HAS_VARIANT) # include <variant> -# define PYBIND11_HAS_VARIANT 1 -# endif -#elif defined(_MSC_VER) && defined(PYBIND11_CPP17) -# include <optional> -# include <variant> -# define PYBIND11_HAS_OPTIONAL 1 -# define PYBIND11_HAS_VARIANT 1 #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -54,29 +38,47 @@ PYBIND11_NAMESPACE_BEGIN(detail) /// Extracts an const lvalue reference or rvalue reference for U based on the type of T (e.g. for /// forwarding a container element). Typically used indirect via forwarded_type(), below. template <typename T, typename U> -using forwarded_type = conditional_t< - std::is_lvalue_reference<T>::value, remove_reference_t<U> &, remove_reference_t<U> &&>; +using forwarded_type = conditional_t<std::is_lvalue_reference<T>::value, + remove_reference_t<U> &, + remove_reference_t<U> &&>; /// Forwards a value U as rvalue or lvalue according to whether T is rvalue or lvalue; typically /// used for forwarding a container's elements. template <typename T, typename U> -forwarded_type<T, U> forward_like(U &&u) { +constexpr forwarded_type<T, U> forward_like(U &&u) { return std::forward<detail::forwarded_type<T, U>>(std::forward<U>(u)); } -template <typename Type, typename Key> struct set_caster { +// Checks if a container has a STL style reserve method. +// This will only return true for a `reserve()` with a `void` return. +template <typename C> +using has_reserve_method = std::is_same<decltype(std::declval<C>().reserve(0)), void>; + +template <typename Type, typename Key> +struct set_caster { using type = Type; using key_conv = make_caster<Key>; +private: + template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0> + void reserve_maybe(const anyset &s, Type *) { + value.reserve(s.size()); + } + void reserve_maybe(const anyset &, void *) {} + +public: bool load(handle src, bool convert) { - if (!isinstance<pybind11::set>(src)) + if (!isinstance<anyset>(src)) { return false; - auto s = reinterpret_borrow<pybind11::set>(src); + } + auto s = reinterpret_borrow<anyset>(src); value.clear(); + reserve_maybe(s, &value); for (auto entry : s) { key_conv conv; - if (!conv.load(entry, convert)) + if (!conv.load(entry, convert)) { return false; + } value.insert(cast_op<Key &&>(std::move(conv))); } return true; @@ -84,35 +86,49 @@ template <typename Type, typename Key> struct set_caster { template <typename T> static handle cast(T &&src, return_value_policy policy, handle parent) { - if (!std::is_lvalue_reference<T>::value) + if (!std::is_lvalue_reference<T>::value) { policy = return_value_policy_override<Key>::policy(policy); + } pybind11::set s; for (auto &&value : src) { - auto value_ = reinterpret_steal<object>(key_conv::cast(forward_like<T>(value), policy, parent)); - if (!value_ || !s.add(value_)) + auto value_ = reinterpret_steal<object>( + key_conv::cast(detail::forward_like<T>(value), policy, parent)); + if (!value_ || !s.add(std::move(value_))) { return handle(); + } } return s.release(); } - PYBIND11_TYPE_CASTER(type, _("Set[") + key_conv::name + _("]")); + PYBIND11_TYPE_CASTER(type, const_name("Set[") + key_conv::name + const_name("]")); }; -template <typename Type, typename Key, typename Value> struct map_caster { - using key_conv = make_caster<Key>; +template <typename Type, typename Key, typename Value> +struct map_caster { + using key_conv = make_caster<Key>; using value_conv = make_caster<Value>; +private: + template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0> + void reserve_maybe(const dict &d, Type *) { + value.reserve(d.size()); + } + void reserve_maybe(const dict &, void *) {} + +public: bool load(handle src, bool convert) { - if (!isinstance<dict>(src)) + if (!isinstance<dict>(src)) { return false; + } auto d = reinterpret_borrow<dict>(src); value.clear(); + reserve_maybe(d, &value); for (auto it : d) { key_conv kconv; value_conv vconv; - if (!kconv.load(it.first.ptr(), convert) || - !vconv.load(it.second.ptr(), convert)) + if (!kconv.load(it.first.ptr(), convert) || !vconv.load(it.second.ptr(), convert)) { return false; + } value.emplace(cast_op<Key &&>(std::move(kconv)), cast_op<Value &&>(std::move(vconv))); } return true; @@ -128,78 +144,92 @@ template <typename Type, typename Key, typename Value> struct map_caster { policy_value = return_value_policy_override<Value>::policy(policy_value); } for (auto &&kv : src) { - auto key = reinterpret_steal<object>(key_conv::cast(forward_like<T>(kv.first), policy_key, parent)); - auto value = reinterpret_steal<object>(value_conv::cast(forward_like<T>(kv.second), policy_value, parent)); - if (!key || !value) + auto key = reinterpret_steal<object>( + key_conv::cast(detail::forward_like<T>(kv.first), policy_key, parent)); + auto value = reinterpret_steal<object>( + value_conv::cast(detail::forward_like<T>(kv.second), policy_value, parent)); + if (!key || !value) { return handle(); - d[key] = value; + } + d[std::move(key)] = std::move(value); } return d.release(); } - PYBIND11_TYPE_CASTER(Type, _("Dict[") + key_conv::name + _(", ") + value_conv::name + _("]")); + PYBIND11_TYPE_CASTER(Type, + const_name("Dict[") + key_conv::name + const_name(", ") + value_conv::name + + const_name("]")); }; -template <typename Type, typename Value> struct list_caster { +template <typename Type, typename Value> +struct list_caster { using value_conv = make_caster<Value>; bool load(handle src, bool convert) { - if (!isinstance<sequence>(src) || isinstance<str>(src)) + if (!isinstance<sequence>(src) || isinstance<bytes>(src) || isinstance<str>(src)) { return false; + } auto s = reinterpret_borrow<sequence>(src); value.clear(); reserve_maybe(s, &value); for (auto it : s) { value_conv conv; - if (!conv.load(it, convert)) + if (!conv.load(it, convert)) { return false; + } value.push_back(cast_op<Value &&>(std::move(conv))); } return true; } private: - template <typename T = Type, - enable_if_t<std::is_same<decltype(std::declval<T>().reserve(0)), void>::value, int> = 0> - void reserve_maybe(sequence s, Type *) { value.reserve(s.size()); } - void reserve_maybe(sequence, void *) { } + template <typename T = Type, enable_if_t<has_reserve_method<T>::value, int> = 0> + void reserve_maybe(const sequence &s, Type *) { + value.reserve(s.size()); + } + void reserve_maybe(const sequence &, void *) {} public: template <typename T> static handle cast(T &&src, return_value_policy policy, handle parent) { - if (!std::is_lvalue_reference<T>::value) + if (!std::is_lvalue_reference<T>::value) { policy = return_value_policy_override<Value>::policy(policy); + } list l(src.size()); - size_t index = 0; + ssize_t index = 0; for (auto &&value : src) { - auto value_ = reinterpret_steal<object>(value_conv::cast(forward_like<T>(value), policy, parent)); - if (!value_) + auto value_ = reinterpret_steal<object>( + value_conv::cast(detail::forward_like<T>(value), policy, parent)); + if (!value_) { return handle(); - PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference + } + PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference } return l.release(); } - PYBIND11_TYPE_CASTER(Type, _("List[") + value_conv::name + _("]")); + PYBIND11_TYPE_CASTER(Type, const_name("List[") + value_conv::name + const_name("]")); }; -template <typename Type, typename Alloc> struct type_caster<std::vector<Type, Alloc>> - : list_caster<std::vector<Type, Alloc>, Type> { }; +template <typename Type, typename Alloc> +struct type_caster<std::vector<Type, Alloc>> : list_caster<std::vector<Type, Alloc>, Type> {}; -template <typename Type, typename Alloc> struct type_caster<std::deque<Type, Alloc>> - : list_caster<std::deque<Type, Alloc>, Type> { }; +template <typename Type, typename Alloc> +struct type_caster<std::deque<Type, Alloc>> : list_caster<std::deque<Type, Alloc>, Type> {}; -template <typename Type, typename Alloc> struct type_caster<std::list<Type, Alloc>> - : list_caster<std::list<Type, Alloc>, Type> { }; +template <typename Type, typename Alloc> +struct type_caster<std::list<Type, Alloc>> : list_caster<std::list<Type, Alloc>, Type> {}; -template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0> struct array_caster { +template <typename ArrayType, typename Value, bool Resizable, size_t Size = 0> +struct array_caster { using value_conv = make_caster<Value>; private: template <bool R = Resizable> bool require_size(enable_if_t<R, size_t> size) { - if (value.size() != size) + if (value.size() != size) { value.resize(size); + } return true; } template <bool R = Resizable> @@ -209,16 +239,19 @@ private: public: bool load(handle src, bool convert) { - if (!isinstance<sequence>(src)) + if (!isinstance<sequence>(src)) { return false; + } auto l = reinterpret_borrow<sequence>(src); - if (!require_size(l.size())) + if (!require_size(l.size())) { return false; + } size_t ctr = 0; for (auto it : l) { value_conv conv; - if (!conv.load(it, convert)) + if (!conv.load(it, convert)) { return false; + } value[ctr++] = cast_op<Value &&>(std::move(conv)); } return true; @@ -227,81 +260,100 @@ public: template <typename T> static handle cast(T &&src, return_value_policy policy, handle parent) { list l(src.size()); - size_t index = 0; + ssize_t index = 0; for (auto &&value : src) { - auto value_ = reinterpret_steal<object>(value_conv::cast(forward_like<T>(value), policy, parent)); - if (!value_) + auto value_ = reinterpret_steal<object>( + value_conv::cast(detail::forward_like<T>(value), policy, parent)); + if (!value_) { return handle(); - PyList_SET_ITEM(l.ptr(), (ssize_t) index++, value_.release().ptr()); // steals a reference + } + PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference } return l.release(); } - PYBIND11_TYPE_CASTER(ArrayType, _("List[") + value_conv::name + _<Resizable>(_(""), _("[") + _<Size>() + _("]")) + _("]")); + PYBIND11_TYPE_CASTER(ArrayType, + const_name<Resizable>(const_name(""), const_name("Annotated[")) + + const_name("List[") + value_conv::name + const_name("]") + + const_name<Resizable>(const_name(""), + const_name(", FixedSize(") + + const_name<Size>() + const_name(")]"))); }; -template <typename Type, size_t Size> struct type_caster<std::array<Type, Size>> - : array_caster<std::array<Type, Size>, Type, false, Size> { }; +template <typename Type, size_t Size> +struct type_caster<std::array<Type, Size>> + : array_caster<std::array<Type, Size>, Type, false, Size> {}; -template <typename Type> struct type_caster<std::valarray<Type>> - : array_caster<std::valarray<Type>, Type, true> { }; +template <typename Type> +struct type_caster<std::valarray<Type>> : array_caster<std::valarray<Type>, Type, true> {}; -template <typename Key, typename Compare, typename Alloc> struct type_caster<std::set<Key, Compare, Alloc>> - : set_caster<std::set<Key, Compare, Alloc>, Key> { }; +template <typename Key, typename Compare, typename Alloc> +struct type_caster<std::set<Key, Compare, Alloc>> + : set_caster<std::set<Key, Compare, Alloc>, Key> {}; -template <typename Key, typename Hash, typename Equal, typename Alloc> struct type_caster<std::unordered_set<Key, Hash, Equal, Alloc>> - : set_caster<std::unordered_set<Key, Hash, Equal, Alloc>, Key> { }; +template <typename Key, typename Hash, typename Equal, typename Alloc> +struct type_caster<std::unordered_set<Key, Hash, Equal, Alloc>> + : set_caster<std::unordered_set<Key, Hash, Equal, Alloc>, Key> {}; -template <typename Key, typename Value, typename Compare, typename Alloc> struct type_caster<std::map<Key, Value, Compare, Alloc>> - : map_caster<std::map<Key, Value, Compare, Alloc>, Key, Value> { }; +template <typename Key, typename Value, typename Compare, typename Alloc> +struct type_caster<std::map<Key, Value, Compare, Alloc>> + : map_caster<std::map<Key, Value, Compare, Alloc>, Key, Value> {}; -template <typename Key, typename Value, typename Hash, typename Equal, typename Alloc> struct type_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>> - : map_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>, Key, Value> { }; +template <typename Key, typename Value, typename Hash, typename Equal, typename Alloc> +struct type_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>> + : map_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>, Key, Value> {}; // This type caster is intended to be used for std::optional and std::experimental::optional -template<typename T> struct optional_caster { - using value_conv = make_caster<typename T::value_type>; +template <typename Type, typename Value = typename Type::value_type> +struct optional_caster { + using value_conv = make_caster<Value>; - template <typename T_> - static handle cast(T_ &&src, return_value_policy policy, handle parent) { - if (!src) - return none().inc_ref(); + template <typename T> + static handle cast(T &&src, return_value_policy policy, handle parent) { + if (!src) { + return none().release(); + } if (!std::is_lvalue_reference<T>::value) { - policy = return_value_policy_override<T>::policy(policy); + policy = return_value_policy_override<Value>::policy(policy); } - return value_conv::cast(*std::forward<T_>(src), policy, parent); + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + return value_conv::cast(*std::forward<T>(src), policy, parent); } bool load(handle src, bool convert) { if (!src) { return false; - } else if (src.is_none()) { - return true; // default-constructed value is already empty + } + if (src.is_none()) { + return true; // default-constructed value is already empty } value_conv inner_caster; - if (!inner_caster.load(src, convert)) + if (!inner_caster.load(src, convert)) { return false; + } - value.emplace(cast_op<typename T::value_type &&>(std::move(inner_caster))); + value.emplace(cast_op<Value &&>(std::move(inner_caster))); return true; } - PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name + _("]")); + PYBIND11_TYPE_CASTER(Type, const_name("Optional[") + value_conv::name + const_name("]")); }; #if defined(PYBIND11_HAS_OPTIONAL) -template<typename T> struct type_caster<std::optional<T>> - : public optional_caster<std::optional<T>> {}; +template <typename T> +struct type_caster<std::optional<T>> : public optional_caster<std::optional<T>> {}; -template<> struct type_caster<std::nullopt_t> - : public void_caster<std::nullopt_t> {}; +template <> +struct type_caster<std::nullopt_t> : public void_caster<std::nullopt_t> {}; #endif #if defined(PYBIND11_HAS_EXP_OPTIONAL) -template<typename T> struct type_caster<std::experimental::optional<T>> +template <typename T> +struct type_caster<std::experimental::optional<T>> : public optional_caster<std::experimental::optional<T>> {}; -template<> struct type_caster<std::experimental::nullopt_t> +template <> +struct type_caster<std::experimental::nullopt_t> : public void_caster<std::experimental::nullopt_t> {}; #endif @@ -322,7 +374,7 @@ struct variant_caster_visitor { /// `namespace::variant` types which provide a `namespace::visit()` function are handled here /// automatically using argument-dependent lookup. Users can provide specializations for other /// variant-like classes, e.g. `boost::variant` and `boost::apply_visitor`. -template <template<typename...> class Variant> +template <template <typename...> class Variant> struct visit_helper { template <typename... Args> static auto call(Args &&...args) -> decltype(visit(std::forward<Args>(args)...)) { @@ -331,9 +383,10 @@ struct visit_helper { }; /// Generic variant caster -template <typename Variant> struct variant_caster; +template <typename Variant> +struct variant_caster; -template <template<typename...> class V, typename... Ts> +template <template <typename...> class V, typename... Ts> struct variant_caster<V<Ts...>> { static_assert(sizeof...(Ts) > 0, "Variant must consist of at least one alternative."); @@ -341,7 +394,7 @@ struct variant_caster<V<Ts...>> { bool load_alternative(handle src, bool convert, type_list<U, Us...>) { auto caster = make_caster<U>(); if (caster.load(src, convert)) { - value = cast_op<U>(caster); + value = cast_op<U>(std::move(caster)); return true; } return load_alternative(src, convert, type_list<Us...>{}); @@ -354,8 +407,9 @@ struct variant_caster<V<Ts...>> { // E.g. `py::int_(1).cast<variant<double, int>>()` needs to fill the `int` // slot of the variant. Without two-pass loading `double` would be filled // because it appears first and a conversion is possible. - if (convert && load_alternative(src, false, type_list<Ts...>{})) + if (convert && load_alternative(src, false, type_list<Ts...>{})) { return true; + } return load_alternative(src, convert, type_list<Ts...>{}); } @@ -366,23 +420,28 @@ struct variant_caster<V<Ts...>> { } using Type = V<Ts...>; - PYBIND11_TYPE_CASTER(Type, _("Union[") + detail::concat(make_caster<Ts>::name...) + _("]")); + PYBIND11_TYPE_CASTER(Type, + const_name("Union[") + detail::concat(make_caster<Ts>::name...) + + const_name("]")); }; #if defined(PYBIND11_HAS_VARIANT) template <typename... Ts> -struct type_caster<std::variant<Ts...>> : variant_caster<std::variant<Ts...>> { }; +struct type_caster<std::variant<Ts...>> : variant_caster<std::variant<Ts...>> {}; + +template <> +struct type_caster<std::monostate> : public void_caster<std::monostate> {}; #endif PYBIND11_NAMESPACE_END(detail) inline std::ostream &operator<<(std::ostream &os, const handle &obj) { +#ifdef PYBIND11_HAS_STRING_VIEW + os << str(obj).cast<std::string_view>(); +#else os << (std::string) str(obj); +#endif return os; } PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h new file mode 100644 index 00000000..e26f4217 --- /dev/null +++ b/include/pybind11/stl/filesystem.h @@ -0,0 +1,116 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#pragma once + +#include "../pybind11.h" +#include "../detail/common.h" +#include "../detail/descr.h" +#include "../cast.h" +#include "../pytypes.h" + +#include <string> + +#ifdef __has_include +# if defined(PYBIND11_CPP17) +# if __has_include(<filesystem>) && \ + PY_VERSION_HEX >= 0x03060000 +# include <filesystem> +# define PYBIND11_HAS_FILESYSTEM 1 +# elif __has_include(<experimental/filesystem>) +# include <experimental/filesystem> +# define PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM 1 +# endif +# endif +#endif + +#if !defined(PYBIND11_HAS_FILESYSTEM) && !defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) \ + && !defined(PYBIND11_HAS_FILESYSTEM_IS_OPTIONAL) +# error \ + "Neither #include <filesystem> nor #include <experimental/filesystem is available. (Use -DPYBIND11_HAS_FILESYSTEM_IS_OPTIONAL to ignore.)" +#endif + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +#if defined(PYBIND11_HAS_FILESYSTEM) || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) +template <typename T> +struct path_caster { + +private: + static PyObject *unicode_from_fs_native(const std::string &w) { +# if !defined(PYPY_VERSION) + return PyUnicode_DecodeFSDefaultAndSize(w.c_str(), ssize_t(w.size())); +# else + // PyPy mistakenly declares the first parameter as non-const. + return PyUnicode_DecodeFSDefaultAndSize(const_cast<char *>(w.c_str()), ssize_t(w.size())); +# endif + } + + static PyObject *unicode_from_fs_native(const std::wstring &w) { + return PyUnicode_FromWideChar(w.c_str(), ssize_t(w.size())); + } + +public: + static handle cast(const T &path, return_value_policy, handle) { + if (auto py_str = unicode_from_fs_native(path.native())) { + return module_::import("pathlib") + .attr("Path")(reinterpret_steal<object>(py_str)) + .release(); + } + return nullptr; + } + + bool load(handle handle, bool) { + // PyUnicode_FSConverter and PyUnicode_FSDecoder normally take care of + // calling PyOS_FSPath themselves, but that's broken on PyPy (PyPy + // issue #3168) so we do it ourselves instead. + PyObject *buf = PyOS_FSPath(handle.ptr()); + if (!buf) { + PyErr_Clear(); + return false; + } + PyObject *native = nullptr; + if constexpr (std::is_same_v<typename T::value_type, char>) { + if (PyUnicode_FSConverter(buf, &native) != 0) { + if (auto *c_str = PyBytes_AsString(native)) { + // AsString returns a pointer to the internal buffer, which + // must not be free'd. + value = c_str; + } + } + } else if constexpr (std::is_same_v<typename T::value_type, wchar_t>) { + if (PyUnicode_FSDecoder(buf, &native) != 0) { + if (auto *c_str = PyUnicode_AsWideCharString(native, nullptr)) { + // AsWideCharString returns a new string that must be free'd. + value = c_str; // Copies the string. + PyMem_Free(c_str); + } + } + } + Py_XDECREF(native); + Py_DECREF(buf); + if (PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + return true; + } + + PYBIND11_TYPE_CASTER(T, const_name("os.PathLike")); +}; + +#endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) + +#if defined(PYBIND11_HAS_FILESYSTEM) +template <> +struct type_caster<std::filesystem::path> : public path_caster<std::filesystem::path> {}; +#elif defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) +template <> +struct type_caster<std::experimental::filesystem::path> + : public path_caster<std::experimental::filesystem::path> {}; +#endif + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 83195ee4..49f1b778 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -10,263 +10,288 @@ #pragma once #include "detail/common.h" +#include "detail/type_caster_base.h" +#include "cast.h" #include "operators.h" #include <algorithm> #include <sstream> +#include <type_traits> PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) /* SFINAE helper class used by 'is_comparable */ -template <typename T> struct container_traits { - template <typename T2> static std::true_type test_comparable(decltype(std::declval<const T2 &>() == std::declval<const T2 &>())*); - template <typename T2> static std::false_type test_comparable(...); - template <typename T2> static std::true_type test_value(typename T2::value_type *); - template <typename T2> static std::false_type test_value(...); - template <typename T2> static std::true_type test_pair(typename T2::first_type *, typename T2::second_type *); - template <typename T2> static std::false_type test_pair(...); - - static constexpr const bool is_comparable = std::is_same<std::true_type, decltype(test_comparable<T>(nullptr))>::value; - static constexpr const bool is_pair = std::is_same<std::true_type, decltype(test_pair<T>(nullptr, nullptr))>::value; - static constexpr const bool is_vector = std::is_same<std::true_type, decltype(test_value<T>(nullptr))>::value; +template <typename T> +struct container_traits { + template <typename T2> + static std::true_type + test_comparable(decltype(std::declval<const T2 &>() == std::declval<const T2 &>()) *); + template <typename T2> + static std::false_type test_comparable(...); + template <typename T2> + static std::true_type test_value(typename T2::value_type *); + template <typename T2> + static std::false_type test_value(...); + template <typename T2> + static std::true_type test_pair(typename T2::first_type *, typename T2::second_type *); + template <typename T2> + static std::false_type test_pair(...); + + static constexpr const bool is_comparable + = std::is_same<std::true_type, decltype(test_comparable<T>(nullptr))>::value; + static constexpr const bool is_pair + = std::is_same<std::true_type, decltype(test_pair<T>(nullptr, nullptr))>::value; + static constexpr const bool is_vector + = std::is_same<std::true_type, decltype(test_value<T>(nullptr))>::value; static constexpr const bool is_element = !is_pair && !is_vector; }; /* Default: is_comparable -> std::false_type */ template <typename T, typename SFINAE = void> -struct is_comparable : std::false_type { }; +struct is_comparable : std::false_type {}; /* For non-map data structures, check whether operator== can be instantiated */ template <typename T> struct is_comparable< - T, enable_if_t<container_traits<T>::is_element && - container_traits<T>::is_comparable>> - : std::true_type { }; + T, + enable_if_t<container_traits<T>::is_element && container_traits<T>::is_comparable>> + : std::true_type {}; -/* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ +/* For a vector/map data structure, recursively check the value type + (which is std::pair for maps) */ template <typename T> -struct is_comparable<T, enable_if_t<container_traits<T>::is_vector>> { - static constexpr const bool value = - is_comparable<typename T::value_type>::value; -}; +struct is_comparable<T, enable_if_t<container_traits<T>::is_vector>> + : is_comparable<typename recursive_container_traits<T>::type_to_check_recursively> {}; + +template <> +struct is_comparable<recursive_bottom> : std::true_type {}; /* For pairs, recursively check the two data types */ template <typename T> struct is_comparable<T, enable_if_t<container_traits<T>::is_pair>> { - static constexpr const bool value = - is_comparable<typename T::first_type>::value && - is_comparable<typename T::second_type>::value; + static constexpr const bool value = is_comparable<typename T::first_type>::value + && is_comparable<typename T::second_type>::value; }; /* Fallback functions */ -template <typename, typename, typename... Args> void vector_if_copy_constructible(const Args &...) { } -template <typename, typename, typename... Args> void vector_if_equal_operator(const Args &...) { } -template <typename, typename, typename... Args> void vector_if_insertion_operator(const Args &...) { } -template <typename, typename, typename... Args> void vector_modifiers(const Args &...) { } +template <typename, typename, typename... Args> +void vector_if_copy_constructible(const Args &...) {} +template <typename, typename, typename... Args> +void vector_if_equal_operator(const Args &...) {} +template <typename, typename, typename... Args> +void vector_if_insertion_operator(const Args &...) {} +template <typename, typename, typename... Args> +void vector_modifiers(const Args &...) {} -template<typename Vector, typename Class_> +template <typename Vector, typename Class_> void vector_if_copy_constructible(enable_if_t<is_copy_constructible<Vector>::value, Class_> &cl) { cl.def(init<const Vector &>(), "Copy constructor"); } -template<typename Vector, typename Class_> +template <typename Vector, typename Class_> void vector_if_equal_operator(enable_if_t<is_comparable<Vector>::value, Class_> &cl) { using T = typename Vector::value_type; cl.def(self == self); cl.def(self != self); - cl.def("count", - [](const Vector &v, const T &x) { - return std::count(v.begin(), v.end(), x); - }, + cl.def( + "count", + [](const Vector &v, const T &x) { return std::count(v.begin(), v.end(), x); }, arg("x"), - "Return the number of times ``x`` appears in the list" - ); + "Return the number of times ``x`` appears in the list"); - cl.def("remove", [](Vector &v, const T &x) { + cl.def( + "remove", + [](Vector &v, const T &x) { auto p = std::find(v.begin(), v.end(), x); - if (p != v.end()) + if (p != v.end()) { v.erase(p); - else + } else { throw value_error(); + } }, arg("x"), "Remove the first item from the list whose value is x. " - "It is an error if there is no such item." - ); + "It is an error if there is no such item."); - cl.def("__contains__", - [](const Vector &v, const T &x) { - return std::find(v.begin(), v.end(), x) != v.end(); - }, + cl.def( + "__contains__", + [](const Vector &v, const T &x) { return std::find(v.begin(), v.end(), x) != v.end(); }, arg("x"), - "Return true the container contains ``x``" - ); + "Return true the container contains ``x``"); } // Vector modifiers -- requires a copyable vector_type: -// (Technically, some of these (pop and __delitem__) don't actually require copyability, but it seems -// silly to allow deletion but not insertion, so include them here too.) +// (Technically, some of these (pop and __delitem__) don't actually require copyability, but it +// seems silly to allow deletion but not insertion, so include them here too.) template <typename Vector, typename Class_> -void vector_modifiers(enable_if_t<is_copy_constructible<typename Vector::value_type>::value, Class_> &cl) { +void vector_modifiers( + enable_if_t<is_copy_constructible<typename Vector::value_type>::value, Class_> &cl) { using T = typename Vector::value_type; using SizeType = typename Vector::size_type; using DiffType = typename Vector::difference_type; auto wrap_i = [](DiffType i, SizeType n) { - if (i < 0) + if (i < 0) { i += n; - if (i < 0 || (SizeType)i >= n) + } + if (i < 0 || (SizeType) i >= n) { throw index_error(); + } return i; }; - cl.def("append", - [](Vector &v, const T &value) { v.push_back(value); }, - arg("x"), - "Add an item to the end of the list"); + cl.def( + "append", + [](Vector &v, const T &value) { v.push_back(value); }, + arg("x"), + "Add an item to the end of the list"); - cl.def(init([](iterable it) { + cl.def(init([](const iterable &it) { auto v = std::unique_ptr<Vector>(new Vector()); v->reserve(len_hint(it)); - for (handle h : it) - v->push_back(h.cast<T>()); + for (handle h : it) { + v->push_back(h.cast<T>()); + } return v.release(); })); - cl.def("clear", - [](Vector &v) { - v.clear(); + cl.def( + "clear", [](Vector &v) { v.clear(); }, "Clear the contents"); + + cl.def( + "extend", + [](Vector &v, const Vector &src) { v.insert(v.end(), src.begin(), src.end()); }, + arg("L"), + "Extend the list by appending all the items in the given list"); + + cl.def( + "extend", + [](Vector &v, const iterable &it) { + const size_t old_size = v.size(); + v.reserve(old_size + len_hint(it)); + try { + for (handle h : it) { + v.push_back(h.cast<T>()); + } + } catch (const cast_error &) { + v.erase(v.begin() + static_cast<typename Vector::difference_type>(old_size), + v.end()); + try { + v.shrink_to_fit(); + } catch (const std::exception &) { + // Do nothing + } + throw; + } }, - "Clear the contents" - ); - - cl.def("extend", - [](Vector &v, const Vector &src) { - v.insert(v.end(), src.begin(), src.end()); - }, - arg("L"), - "Extend the list by appending all the items in the given list" - ); - - cl.def("extend", - [](Vector &v, iterable it) { - const size_t old_size = v.size(); - v.reserve(old_size + len_hint(it)); - try { - for (handle h : it) { - v.push_back(h.cast<T>()); - } - } catch (const cast_error &) { - v.erase(v.begin() + static_cast<typename Vector::difference_type>(old_size), v.end()); - try { - v.shrink_to_fit(); - } catch (const std::exception &) { - // Do nothing - } - throw; - } - }, - arg("L"), - "Extend the list by appending all the items in the given list" - ); + arg("L"), + "Extend the list by appending all the items in the given list"); - cl.def("insert", + cl.def( + "insert", [](Vector &v, DiffType i, const T &x) { // Can't use wrap_i; i == v.size() is OK - if (i < 0) + if (i < 0) { i += v.size(); - if (i < 0 || (SizeType)i > v.size()) + } + if (i < 0 || (SizeType) i > v.size()) { throw index_error(); + } v.insert(v.begin() + i, x); }, - arg("i") , arg("x"), - "Insert an item at a given position." - ); + arg("i"), + arg("x"), + "Insert an item at a given position."); - cl.def("pop", + cl.def( + "pop", [](Vector &v) { - if (v.empty()) + if (v.empty()) { throw index_error(); - T t = v.back(); + } + T t = std::move(v.back()); v.pop_back(); return t; }, - "Remove and return the last item" - ); + "Remove and return the last item"); - cl.def("pop", + cl.def( + "pop", [wrap_i](Vector &v, DiffType i) { i = wrap_i(i, v.size()); - T t = v[(SizeType) i]; - v.erase(v.begin() + i); + T t = std::move(v[(SizeType) i]); + v.erase(std::next(v.begin(), i)); return t; }, arg("i"), - "Remove and return the item at index ``i``" - ); + "Remove and return the item at index ``i``"); - cl.def("__setitem__", - [wrap_i](Vector &v, DiffType i, const T &t) { - i = wrap_i(i, v.size()); - v[(SizeType)i] = t; - } - ); + cl.def("__setitem__", [wrap_i](Vector &v, DiffType i, const T &t) { + i = wrap_i(i, v.size()); + v[(SizeType) i] = t; + }); /// Slicing protocol - cl.def("__getitem__", - [](const Vector &v, slice slice) -> Vector * { - size_t start, stop, step, slicelength; + cl.def( + "__getitem__", + [](const Vector &v, const slice &slice) -> Vector * { + size_t start = 0, stop = 0, step = 0, slicelength = 0; - if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) + if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) { throw error_already_set(); + } auto *seq = new Vector(); seq->reserve((size_t) slicelength); - for (size_t i=0; i<slicelength; ++i) { + for (size_t i = 0; i < slicelength; ++i) { seq->push_back(v[start]); start += step; } return seq; }, arg("s"), - "Retrieve list elements using a slice object" - ); + "Retrieve list elements using a slice object"); - cl.def("__setitem__", - [](Vector &v, slice slice, const Vector &value) { - size_t start, stop, step, slicelength; - if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) + cl.def( + "__setitem__", + [](Vector &v, const slice &slice, const Vector &value) { + size_t start = 0, stop = 0, step = 0, slicelength = 0; + if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) { throw error_already_set(); + } - if (slicelength != value.size()) - throw std::runtime_error("Left and right hand size of slice assignment have different sizes!"); + if (slicelength != value.size()) { + throw std::runtime_error( + "Left and right hand size of slice assignment have different sizes!"); + } - for (size_t i=0; i<slicelength; ++i) { + for (size_t i = 0; i < slicelength; ++i) { v[start] = value[i]; start += step; } }, - "Assign list elements using a slice object" - ); + "Assign list elements using a slice object"); - cl.def("__delitem__", + cl.def( + "__delitem__", [wrap_i](Vector &v, DiffType i) { i = wrap_i(i, v.size()); v.erase(v.begin() + i); }, - "Delete the list elements at index ``i``" - ); + "Delete the list elements at index ``i``"); - cl.def("__delitem__", - [](Vector &v, slice slice) { - size_t start, stop, step, slicelength; + cl.def( + "__delitem__", + [](Vector &v, const slice &slice) { + size_t start = 0, stop = 0, step = 0, slicelength = 0; - if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) + if (!slice.compute(v.size(), &start, &stop, &step, &slicelength)) { throw error_already_set(); + } if (step == 1 && false) { v.erase(v.begin() + (DiffType) start, v.begin() + DiffType(start + slicelength)); @@ -277,15 +302,15 @@ void vector_modifiers(enable_if_t<is_copy_constructible<typename Vector::value_t } } }, - "Delete list elements using a slice object" - ); - + "Delete list elements using a slice object"); } // If the type has an operator[] that doesn't return a reference (most notably std::vector<bool>), // we have to access by copying; otherwise we return by reference. -template <typename Vector> using vector_needs_copy = negation< - std::is_same<decltype(std::declval<Vector>()[typename Vector::size_type()]), typename Vector::value_type &>>; +template <typename Vector> +using vector_needs_copy + = negation<std::is_same<decltype(std::declval<Vector>()[typename Vector::size_type()]), + typename Vector::value_type &>>; // The usual case: access and iterate by reference template <typename Vector, typename Class_> @@ -293,31 +318,34 @@ void vector_accessor(enable_if_t<!vector_needs_copy<Vector>::value, Class_> &cl) using T = typename Vector::value_type; using SizeType = typename Vector::size_type; using DiffType = typename Vector::difference_type; - using ItType = typename Vector::iterator; + using ItType = typename Vector::iterator; auto wrap_i = [](DiffType i, SizeType n) { - if (i < 0) + if (i < 0) { i += n; - if (i < 0 || (SizeType)i >= n) + } + if (i < 0 || (SizeType) i >= n) { throw index_error(); + } return i; }; - cl.def("__getitem__", + cl.def( + "__getitem__", [wrap_i](Vector &v, DiffType i) -> T & { i = wrap_i(i, v.size()); - return v[(SizeType)i]; + return v[(SizeType) i]; }, return_value_policy::reference_internal // ref + keepalive ); - cl.def("__iter__", - [](Vector &v) { - return make_iterator< - return_value_policy::reference_internal, ItType, ItType, T&>( - v.begin(), v.end()); - }, - keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ + cl.def( + "__iter__", + [](Vector &v) { + return make_iterator<return_value_policy::reference_internal, ItType, ItType, T &>( + v.begin(), v.end()); + }, + keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ ); } @@ -327,53 +355,64 @@ void vector_accessor(enable_if_t<vector_needs_copy<Vector>::value, Class_> &cl) using T = typename Vector::value_type; using SizeType = typename Vector::size_type; using DiffType = typename Vector::difference_type; - using ItType = typename Vector::iterator; - cl.def("__getitem__", - [](const Vector &v, DiffType i) -> T { - if (i < 0 && (i += v.size()) < 0) - throw index_error(); - if ((SizeType)i >= v.size()) + using ItType = typename Vector::iterator; + cl.def("__getitem__", [](const Vector &v, DiffType i) -> T { + if (i < 0) { + i += v.size(); + if (i < 0) { throw index_error(); - return v[(SizeType)i]; + } } - ); + auto i_st = static_cast<SizeType>(i); + if (i_st >= v.size()) { + throw index_error(); + } + return v[i_st]; + }); - cl.def("__iter__", - [](Vector &v) { - return make_iterator< - return_value_policy::copy, ItType, ItType, T>( - v.begin(), v.end()); - }, - keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ + cl.def( + "__iter__", + [](Vector &v) { + return make_iterator<return_value_policy::copy, ItType, ItType, T>(v.begin(), v.end()); + }, + keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ ); } -template <typename Vector, typename Class_> auto vector_if_insertion_operator(Class_ &cl, std::string const &name) - -> decltype(std::declval<std::ostream&>() << std::declval<typename Vector::value_type>(), void()) { +template <typename Vector, typename Class_> +auto vector_if_insertion_operator(Class_ &cl, std::string const &name) + -> decltype(std::declval<std::ostream &>() << std::declval<typename Vector::value_type>(), + void()) { using size_type = typename Vector::size_type; - cl.def("__repr__", - [name](Vector &v) { + cl.def( + "__repr__", + [name](Vector &v) { std::ostringstream s; s << name << '['; - for (size_type i=0; i < v.size(); ++i) { + for (size_type i = 0; i < v.size(); ++i) { s << v[i]; - if (i != v.size() - 1) + if (i != v.size() - 1) { s << ", "; + } } s << ']'; return s.str(); }, - "Return the canonical string representation of this list." - ); + "Return the canonical string representation of this list."); } // Provide the buffer interface for vectors if we have data() and we have a format for it -// GCC seems to have "void std::vector<bool>::data()" - doing SFINAE on the existence of data() is insufficient, we need to check it returns an appropriate pointer +// GCC seems to have "void std::vector<bool>::data()" - doing SFINAE on the existence of data() +// is insufficient, we need to check it returns an appropriate pointer template <typename Vector, typename = void> struct vector_has_data_and_format : std::false_type {}; template <typename Vector> -struct vector_has_data_and_format<Vector, enable_if_t<std::is_same<decltype(format_descriptor<typename Vector::value_type>::format(), std::declval<Vector>().data()), typename Vector::value_type*>::value>> : std::true_type {}; +struct vector_has_data_and_format< + Vector, + enable_if_t<std::is_same<decltype(format_descriptor<typename Vector::value_type>::format(), + std::declval<Vector>().data()), + typename Vector::value_type *>::value>> : std::true_type {}; // [workaround(intel)] Separate function required here // Workaround as the Intel compiler does not compile the enable_if_t part below @@ -388,49 +427,60 @@ constexpr bool args_any_are_buffer() { // Add the buffer interface to a vector template <typename Vector, typename Class_, typename... Args> -void vector_buffer_impl(Class_& cl, std::true_type) { +void vector_buffer_impl(Class_ &cl, std::true_type) { using T = typename Vector::value_type; - static_assert(vector_has_data_and_format<Vector>::value, "There is not an appropriate format descriptor for this vector"); + static_assert(vector_has_data_and_format<Vector>::value, + "There is not an appropriate format descriptor for this vector"); - // numpy.h declares this for arbitrary types, but it may raise an exception and crash hard at runtime if PYBIND11_NUMPY_DTYPE hasn't been called, so check here + // numpy.h declares this for arbitrary types, but it may raise an exception and crash hard + // at runtime if PYBIND11_NUMPY_DTYPE hasn't been called, so check here format_descriptor<T>::format(); - cl.def_buffer([](Vector& v) -> buffer_info { - return buffer_info(v.data(), static_cast<ssize_t>(sizeof(T)), format_descriptor<T>::format(), 1, {v.size()}, {sizeof(T)}); + cl.def_buffer([](Vector &v) -> buffer_info { + return buffer_info(v.data(), + static_cast<ssize_t>(sizeof(T)), + format_descriptor<T>::format(), + 1, + {v.size()}, + {sizeof(T)}); }); - cl.def(init([](buffer buf) { + cl.def(init([](const buffer &buf) { auto info = buf.request(); - if (info.ndim != 1 || info.strides[0] % static_cast<ssize_t>(sizeof(T))) + if (info.ndim != 1 || info.strides[0] % static_cast<ssize_t>(sizeof(T))) { throw type_error("Only valid 1D buffers can be copied to a vector"); - if (!detail::compare_buffer_info<T>::compare(info) || (ssize_t) sizeof(T) != info.itemsize) - throw type_error("Format mismatch (Python: " + info.format + " C++: " + format_descriptor<T>::format() + ")"); + } + if (!detail::compare_buffer_info<T>::compare(info) + || (ssize_t) sizeof(T) != info.itemsize) { + throw type_error("Format mismatch (Python: " + info.format + + " C++: " + format_descriptor<T>::format() + ")"); + } - T *p = static_cast<T*>(info.ptr); + T *p = static_cast<T *>(info.ptr); ssize_t step = info.strides[0] / static_cast<ssize_t>(sizeof(T)); T *end = p + info.shape[0] * step; if (step == 1) { return Vector(p, end); } - else { - Vector vec; - vec.reserve((size_t) info.shape[0]); - for (; p != end; p += step) - vec.push_back(*p); - return vec; + Vector vec; + vec.reserve((size_t) info.shape[0]); + for (; p != end; p += step) { + vec.push_back(*p); } + return vec; })); return; } template <typename Vector, typename Class_, typename... Args> -void vector_buffer_impl(Class_&, std::false_type) {} +void vector_buffer_impl(Class_ &, std::false_type) {} template <typename Vector, typename Class_, typename... Args> -void vector_buffer(Class_& cl) { - vector_buffer_impl<Vector, Class_, Args...>(cl, detail::any_of<std::is_same<Args, buffer_protocol>...>{}); +void vector_buffer(Class_ &cl) { + vector_buffer_impl<Vector, Class_, Args...>( + cl, detail::any_of<std::is_same<Args, buffer_protocol>...>{}); } PYBIND11_NAMESPACE_END(detail) @@ -439,13 +489,13 @@ PYBIND11_NAMESPACE_END(detail) // std::vector // template <typename Vector, typename holder_type = std::unique_ptr<Vector>, typename... Args> -class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, Args&&... args) { +class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, Args &&...args) { using Class_ = class_<Vector, holder_type>; // If the value_type is unregistered (e.g. a converting type) or is itself registered // module-local then make the vector binding module-local as well: using vtype = typename Vector::value_type; - auto vtype_info = detail::get_type_info(typeid(vtype)); + auto *vtype_info = detail::get_type_info(typeid(vtype)); bool local = !vtype_info || vtype_info->module_local; Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward<Args>(args)...); @@ -470,18 +520,13 @@ class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, A // Accessor and iterator; return by value if copyable, otherwise we return by ref + keep-alive detail::vector_accessor<Vector, Class_>(cl); - cl.def("__bool__", - [](const Vector &v) -> bool { - return !v.empty(); - }, - "Check whether the list is nonempty" - ); + cl.def( + "__bool__", + [](const Vector &v) -> bool { return !v.empty(); }, + "Check whether the list is nonempty"); cl.def("__len__", &Vector::size); - - - #if 0 // C++ style functions deprecated, leaving it here as an example cl.def(init<size_type>()); @@ -525,8 +570,6 @@ class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, A return cl; } - - // // std::map, std::unordered_map // @@ -534,81 +577,140 @@ class_<Vector, holder_type> bind_vector(handle scope, std::string const &name, A PYBIND11_NAMESPACE_BEGIN(detail) /* Fallback functions */ -template <typename, typename, typename... Args> void map_if_insertion_operator(const Args &...) { } -template <typename, typename, typename... Args> void map_assignment(const Args &...) { } +template <typename, typename, typename... Args> +void map_if_insertion_operator(const Args &...) {} +template <typename, typename, typename... Args> +void map_assignment(const Args &...) {} // Map assignment when copy-assignable: just copy the value template <typename Map, typename Class_> -void map_assignment(enable_if_t<is_copy_assignable<typename Map::mapped_type>::value, Class_> &cl) { +void map_assignment( + enable_if_t<is_copy_assignable<typename Map::mapped_type>::value, Class_> &cl) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - cl.def("__setitem__", - [](Map &m, const KeyType &k, const MappedType &v) { - auto it = m.find(k); - if (it != m.end()) it->second = v; - else m.emplace(k, v); - } - ); + cl.def("__setitem__", [](Map &m, const KeyType &k, const MappedType &v) { + auto it = m.find(k); + if (it != m.end()) { + it->second = v; + } else { + m.emplace(k, v); + } + }); } -// Not copy-assignable, but still copy-constructible: we can update the value by erasing and reinserting -template<typename Map, typename Class_> -void map_assignment(enable_if_t< - !is_copy_assignable<typename Map::mapped_type>::value && - is_copy_constructible<typename Map::mapped_type>::value, - Class_> &cl) { +// Not copy-assignable, but still copy-constructible: we can update the value by erasing and +// reinserting +template <typename Map, typename Class_> +void map_assignment(enable_if_t<!is_copy_assignable<typename Map::mapped_type>::value + && is_copy_constructible<typename Map::mapped_type>::value, + Class_> &cl) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - cl.def("__setitem__", - [](Map &m, const KeyType &k, const MappedType &v) { - // We can't use m[k] = v; because value type might not be default constructable - auto r = m.emplace(k, v); - if (!r.second) { - // value type is not copy assignable so the only way to insert it is to erase it first... - m.erase(r.first); - m.emplace(k, v); - } - } - ); + cl.def("__setitem__", [](Map &m, const KeyType &k, const MappedType &v) { + // We can't use m[k] = v; because value type might not be default constructable + auto r = m.emplace(k, v); + if (!r.second) { + // value type is not copy assignable so the only way to insert it is to erase it + // first... + m.erase(r.first); + m.emplace(k, v); + } + }); } - -template <typename Map, typename Class_> auto map_if_insertion_operator(Class_ &cl, std::string const &name) --> decltype(std::declval<std::ostream&>() << std::declval<typename Map::key_type>() << std::declval<typename Map::mapped_type>(), void()) { - - cl.def("__repr__", - [name](Map &m) { +template <typename Map, typename Class_> +auto map_if_insertion_operator(Class_ &cl, std::string const &name) + -> decltype(std::declval<std::ostream &>() << std::declval<typename Map::key_type>() + << std::declval<typename Map::mapped_type>(), + void()) { + + cl.def( + "__repr__", + [name](Map &m) { std::ostringstream s; s << name << '{'; bool f = false; for (auto const &kv : m) { - if (f) + if (f) { s << ", "; + } s << kv.first << ": " << kv.second; f = true; } s << '}'; return s.str(); }, - "Return the canonical string representation of this map." - ); + "Return the canonical string representation of this map."); } +template <typename KeyType> +struct keys_view { + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual bool contains(const KeyType &k) = 0; + virtual bool contains(const object &k) = 0; + virtual ~keys_view() = default; +}; + +template <typename MappedType> +struct values_view { + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~values_view() = default; +}; + +template <typename KeyType, typename MappedType> +struct items_view { + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~items_view() = default; +}; + +template <typename Map, typename KeysView> +struct KeysViewImpl : public KeysView { + explicit KeysViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_key_iterator(map.begin(), map.end()); } + bool contains(const typename Map::key_type &k) override { return map.find(k) != map.end(); } + bool contains(const object &) override { return false; } + Map ↦ +}; + +template <typename Map, typename ValuesView> +struct ValuesViewImpl : public ValuesView { + explicit ValuesViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_value_iterator(map.begin(), map.end()); } + Map ↦ +}; + +template <typename Map, typename ItemsView> +struct ItemsViewImpl : public ItemsView { + explicit ItemsViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_iterator(map.begin(), map.end()); } + Map ↦ +}; PYBIND11_NAMESPACE_END(detail) template <typename Map, typename holder_type = std::unique_ptr<Map>, typename... Args> -class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args&&... args) { +class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; + using StrippedKeyType = detail::remove_cvref_t<KeyType>; + using StrippedMappedType = detail::remove_cvref_t<MappedType>; + using KeysView = detail::keys_view<StrippedKeyType>; + using ValuesView = detail::values_view<StrippedMappedType>; + using ItemsView = detail::items_view<StrippedKeyType, StrippedMappedType>; using Class_ = class_<Map, holder_type>; // If either type is a non-module-local bound type then make the map binding non-local as well; // otherwise (e.g. both types are either module-local or converting) the map will be // module-local. - auto tinfo = detail::get_type_info(typeid(MappedType)); + auto *tinfo = detail::get_type_info(typeid(MappedType)); bool local = !tinfo || tinfo->module_local; if (local) { tinfo = detail::get_type_info(typeid(KeyType)); @@ -616,57 +718,130 @@ class_<Map, holder_type> bind_map(handle scope, const std::string &name, Args&&. } Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward<Args>(args)...); + static constexpr auto key_type_descr = detail::make_caster<KeyType>::name; + static constexpr auto mapped_type_descr = detail::make_caster<MappedType>::name; + std::string key_type_name(key_type_descr.text), mapped_type_name(mapped_type_descr.text); + + // If key type isn't properly wrapped, fall back to C++ names + if (key_type_name == "%") { + key_type_name = detail::type_info_description(typeid(KeyType)); + } + // Similarly for value type: + if (mapped_type_name == "%") { + mapped_type_name = detail::type_info_description(typeid(MappedType)); + } + + // Wrap KeysView[KeyType] if it wasn't already wrapped + if (!detail::get_type_info(typeid(KeysView))) { + class_<KeysView> keys_view( + scope, ("KeysView[" + key_type_name + "]").c_str(), pybind11::module_local(local)); + keys_view.def("__len__", &KeysView::len); + keys_view.def("__iter__", + &KeysView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + keys_view.def("__contains__", + static_cast<bool (KeysView::*)(const KeyType &)>(&KeysView::contains)); + // Fallback for when the object is not of the key type + keys_view.def("__contains__", + static_cast<bool (KeysView::*)(const object &)>(&KeysView::contains)); + } + // Similarly for ValuesView: + if (!detail::get_type_info(typeid(ValuesView))) { + class_<ValuesView> values_view(scope, + ("ValuesView[" + mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + values_view.def("__len__", &ValuesView::len); + values_view.def("__iter__", + &ValuesView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } + // Similarly for ItemsView: + if (!detail::get_type_info(typeid(ItemsView))) { + class_<ItemsView> items_view( + scope, + ("ItemsView[" + key_type_name + ", ").append(mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + items_view.def("__len__", &ItemsView::len); + items_view.def("__iter__", + &ItemsView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } cl.def(init<>()); // Register stream insertion operator (if possible) detail::map_if_insertion_operator<Map, Class_>(cl, name); - cl.def("__bool__", + cl.def( + "__bool__", [](const Map &m) -> bool { return !m.empty(); }, - "Check whether the map is nonempty" + "Check whether the map is nonempty"); + + cl.def( + "__iter__", + [](Map &m) { return make_key_iterator(m.begin(), m.end()); }, + keep_alive<0, 1>() /* Essential: keep map alive while iterator exists */ ); - cl.def("__iter__", - [](Map &m) { return make_key_iterator(m.begin(), m.end()); }, - keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ + cl.def( + "keys", + [](Map &m) { + return std::unique_ptr<KeysView>(new detail::KeysViewImpl<Map, KeysView>(m)); + }, + keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); - cl.def("items", - [](Map &m) { return make_iterator(m.begin(), m.end()); }, - keep_alive<0, 1>() /* Essential: keep list alive while iterator exists */ + cl.def( + "values", + [](Map &m) { + return std::unique_ptr<ValuesView>(new detail::ValuesViewImpl<Map, ValuesView>(m)); + }, + keep_alive<0, 1>() /* Essential: keep map alive while view exists */ + ); + + cl.def( + "items", + [](Map &m) { + return std::unique_ptr<ItemsView>(new detail::ItemsViewImpl<Map, ItemsView>(m)); + }, + keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); - cl.def("__getitem__", + cl.def( + "__getitem__", [](Map &m, const KeyType &k) -> MappedType & { auto it = m.find(k); - if (it == m.end()) - throw key_error(); - return it->second; + if (it == m.end()) { + throw key_error(); + } + return it->second; }, return_value_policy::reference_internal // ref + keepalive ); - cl.def("__contains__", - [](Map &m, const KeyType &k) -> bool { - auto it = m.find(k); - if (it == m.end()) - return false; - return true; + cl.def("__contains__", [](Map &m, const KeyType &k) -> bool { + auto it = m.find(k); + if (it == m.end()) { + return false; } - ); + return true; + }); + // Fallback for when the object is not of the key type + cl.def("__contains__", [](Map &, const object &) -> bool { return false; }); // Assignment provided only if the type is copyable detail::map_assignment<Map, Class_>(cl); - cl.def("__delitem__", - [](Map &m, const KeyType &k) { - auto it = m.find(k); - if (it == m.end()) - throw key_error(); - m.erase(it); - } - ); + cl.def("__delitem__", [](Map &m, const KeyType &k) { + auto it = m.find(k); + if (it == m.end()) { + throw key_error(); + } + m.erase(it); + }); cl.def("__len__", &Map::size); diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 00000000..aa914f9e --- /dev/null +++ b/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster<PyObject> { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template <typename T, + detail::enable_if_t<!is_same_ignoring_cvref<T, PyObject *>::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref<T, PyObject *>::value, + "Invalid C++ type T for to-Python conversion (type_caster<PyObject>)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster<PyObject>::cast(): unsupported return_value_policy: " + + std::to_string(static_cast<int>(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow<object>(src); + return true; + } + + template <typename T> + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..021ced24 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,107 @@ +import os + +import nox + +nox.needs_version = ">=2022.1.7" +nox.options.sessions = ["lint", "tests", "tests_packaging"] + +PYTHON_VERSIONS = [ + "3.6", + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "pypy3.7", + "pypy3.8", + "pypy3.9", +] + +if os.environ.get("CI", None): + nox.options.error_on_missing_interpreters = True + + +@nox.session(reuse_venv=True) +def lint(session: nox.Session) -> None: + """ + Lint the codebase (except for clang-format/tidy). + """ + session.install("pre-commit") + session.run("pre-commit", "run", "-a", *session.posargs) + + +@nox.session(python=PYTHON_VERSIONS) +def tests(session: nox.Session) -> None: + """ + Run the tests (requires a compiler). + """ + tmpdir = session.create_tmp() + session.install("cmake") + session.install("-r", "tests/requirements.txt") + session.run( + "cmake", + "-S.", + f"-B{tmpdir}", + "-DPYBIND11_WERROR=ON", + "-DDOWNLOAD_CATCH=ON", + "-DDOWNLOAD_EIGEN=ON", + *session.posargs, + ) + session.run("cmake", "--build", tmpdir) + session.run("cmake", "--build", tmpdir, "--config=Release", "--target", "check") + + +@nox.session +def tests_packaging(session: nox.Session) -> None: + """ + Run the packaging tests. + """ + + session.install("-r", "tests/requirements.txt", "--prefer-binary") + session.run("pytest", "tests/extra_python_package", *session.posargs) + + +@nox.session(reuse_venv=True) +def docs(session: nox.Session) -> None: + """ + Build the docs. Pass "serve" to serve. + """ + + session.install("-r", "docs/requirements.txt") + session.chdir("docs") + + if "pdf" in session.posargs: + session.run("sphinx-build", "-M", "latexpdf", ".", "_build") + return + + session.run("sphinx-build", "-M", "html", ".", "_build") + + if "serve" in session.posargs: + session.log("Launching docs at http://localhost:8000/ - use Ctrl-C to quit") + session.run("python", "-m", "http.server", "8000", "-d", "_build/html") + elif session.posargs: + session.error("Unsupported argument to docs") + + +@nox.session(reuse_venv=True) +def make_changelog(session: nox.Session) -> None: + """ + Inspect the closed issues and make entries for a changelog. + """ + session.install("ghapi", "rich") + session.run("python", "tools/make_changelog.py") + + +@nox.session(reuse_venv=True) +def build(session: nox.Session) -> None: + """ + Build SDists and wheels. + """ + + session.install("build") + session.log("Building normal files") + session.run("python", "-m", "build", *session.posargs) + session.log("Building pybind11-global files (PYBIND11_GLOBAL_SDIST=1)") + session.run( + "python", "-m", "build", *session.posargs, env={"PYBIND11_GLOBAL_SDIST": "1"} + ) diff --git a/pybind11/__init__.py b/pybind11/__init__.py index ad654208..7c10b305 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -1,12 +1,17 @@ -# -*- coding: utf-8 -*- +import sys -from ._version import version_info, __version__ -from .commands import get_include, get_cmake_dir +if sys.version_info < (3, 6): # noqa: UP036 + msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5." + raise ImportError(msg) +from ._version import __version__, version_info +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir + __all__ = ( "version_info", "__version__", "get_include", "get_cmake_dir", + "get_pkgconfig_dir", ) diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 020988c6..180665c2 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- -from __future__ import print_function +# pylint: disable=missing-function-docstring import argparse import sys import sysconfig -from .commands import get_include, get_cmake_dir +from ._version import __version__ +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir -def print_includes(): - # type: () -> None +def print_includes() -> None: dirs = [ sysconfig.get_path("include"), sysconfig.get_path("platinclude"), @@ -25,11 +24,15 @@ def print_includes(): print(" ".join("-I" + d for d in unique_dirs)) -def main(): - # type: () -> None - +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) + parser.add_argument( "--includes", action="store_true", help="Include flags for both pybind11 and Python headers.", @@ -39,6 +42,11 @@ def main(): action="store_true", help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.", ) + parser.add_argument( + "--pkgconfigdir", + action="store_true", + help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() @@ -46,6 +54,8 @@ def main(): print_includes() if args.cmakedir: print(get_cmake_dir()) + if args.pkgconfigdir: + print(get_pkgconfig_dir()) if __name__ == "__main__": diff --git a/pybind11/_version.py b/pybind11/_version.py index f8b795ee..1c310e0c 100644 --- a/pybind11/_version.py +++ b/pybind11/_version.py @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- +from typing import Union -def _to_int(s): +def _to_int(s: str) -> Union[int, str]: try: return int(s) except ValueError: return s -__version__ = "2.6.2" +__version__ = "2.11.0" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/pybind11/_version.pyi b/pybind11/_version.pyi deleted file mode 100644 index 970184c7..00000000 --- a/pybind11/_version.pyi +++ /dev/null @@ -1,6 +0,0 @@ -from typing import Union, Tuple - -def _to_int(s: str) -> Union[int, str]: ... - -__version__: str -version_info: Tuple[Union[int, str], ...] diff --git a/pybind11/commands.py b/pybind11/commands.py index 34dbaf8a..b11690f4 100644 --- a/pybind11/commands.py +++ b/pybind11/commands.py @@ -1,22 +1,37 @@ -# -*- coding: utf-8 -*- import os - DIR = os.path.abspath(os.path.dirname(__file__)) -def get_include(user=False): - # type: (bool) -> str +def get_include(user: bool = False) -> str: # noqa: ARG001 + """ + Return the path to the pybind11 include directory. The historical "user" + argument is unused, and may be removed. + """ installed_path = os.path.join(DIR, "include") source_path = os.path.join(os.path.dirname(DIR), "include") return installed_path if os.path.exists(installed_path) else source_path -def get_cmake_dir(): - # type: () -> str +def get_cmake_dir() -> str: + """ + Return the path to the pybind11 CMake module directory. + """ cmake_installed_path = os.path.join(DIR, "share", "cmake", "pybind11") if os.path.exists(cmake_installed_path): return cmake_installed_path - else: - msg = "pybind11 not installed, installation required to access the CMake files" - raise ImportError(msg) + + msg = "pybind11 not installed, installation required to access the CMake files" + raise ImportError(msg) + + +def get_pkgconfig_dir() -> str: + """ + Return the path to the pybind11 pkgconfig directory. + """ + pkgconfig_installed_path = os.path.join(DIR, "share", "pkgconfig") + if os.path.exists(pkgconfig_installed_path): + return pkgconfig_installed_path + + msg = "pybind11 not installed, installation required to access the pkgconfig files" + raise ImportError(msg) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index c69064ca..aeeee9dc 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ This module provides helpers for C++11+ projects using pybind11. @@ -41,26 +39,40 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import contextlib import os +import platform +import shlex import shutil import sys +import sysconfig import tempfile import threading -import platform import warnings +from functools import lru_cache +from pathlib import Path +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + TypeVar, + Union, +) try: - from setuptools.command.build_ext import build_ext as _build_ext from setuptools import Extension as _Extension + from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import build_ext as _build_ext # type: ignore[assignment] + from distutils.extension import Extension as _Extension # type: ignore[assignment] -import distutils.errors import distutils.ccompiler +import distutils.errors - -WIN = sys.platform.startswith("win32") -PY2 = sys.version_info[0] < 3 +WIN = sys.platform.startswith("win32") and "mingw" not in sysconfig.get_platform() MACOS = sys.platform.startswith("darwin") STD_TMPL = "/std:c++{}" if WIN else "-std=c++{}" @@ -84,7 +96,7 @@ class Pybind11Extension(_Extension): * ``stdlib=libc++`` on macOS * ``visibility=hidden`` and ``-g0`` on Unix - Finally, you can set ``cxx_std`` via constructor or afterwords to enable + Finally, you can set ``cxx_std`` via constructor or afterwards to enable flags for C++ std, and a few extra helper flags related to the C++ standard level. It is _highly_ recommended you either set this, or use the provided ``build_ext``, which will search for the highest supported extension for @@ -94,22 +106,18 @@ class Pybind11Extension(_Extension): If you want to add pybind11 headers manually, for example for an exact git checkout, then set ``include_pybind11=False``. - - Warning: do not use property-based access to the instance on Python 2 - - this is an ugly old-style class due to Distutils. """ # flags are prepended, so that they can be further overridden, e.g. by # ``extra_compile_args=["-g"]``. - def _add_cflags(self, flags): + def _add_cflags(self, flags: List[str]) -> None: self.extra_compile_args[:0] = flags - def _add_ldflags(self, flags): + def _add_ldflags(self, flags: List[str]) -> None: self.extra_link_args[:0] = flags - def __init__(self, *args, **kwargs): - + def __init__(self, *args: Any, **kwargs: Any) -> None: self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -118,9 +126,7 @@ class Pybind11Extension(_Extension): include_pybind11 = kwargs.pop("include_pybind11", True) - # Can't use super here because distutils has old-style classes in - # Python 2! - _Extension.__init__(self, *args, **kwargs) + super().__init__(*args, **kwargs) # Include the installed package pybind11 headers if include_pybind11: @@ -132,40 +138,40 @@ class Pybind11Extension(_Extension): if pyinc not in self.include_dirs: self.include_dirs.append(pyinc) - except ImportError: + except ModuleNotFoundError: pass - # Have to use the accessor manually to support Python 2 distutils - Pybind11Extension.cxx_std.__set__(self, cxx_std) + self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: - cflags += ["-fvisibility=hidden", "-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] + cflags += ["-fvisibility=hidden"] + env_cflags = os.environ.get("CFLAGS", "") + env_cppflags = os.environ.get("CPPFLAGS", "") + c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) + if not any(opt.startswith("-g") for opt in c_cpp_flags): + cflags += ["-g0"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property - def cxx_std(self): + def cxx_std(self) -> int: """ - The CXX standard level. If set, will add the required flags. If left - at 0, it will trigger an automatic search when pybind11's build_ext - is used. If None, will have no effect. Besides just the flags, this - may add a register warning/error fix for Python 2 or macos-min 10.9 - or 10.14. + The CXX standard level. If set, will add the required flags. If left at + 0, it will trigger an automatic search when pybind11's build_ext is + used. If None, will have no effect. Besides just the flags, this may + add a macos-min 10.9 or 10.14 flag if MACOSX_DEPLOYMENT_TARGET is + unset. """ return self._cxx_level @cxx_std.setter - def cxx_std(self, level): - + def cxx_std(self, level: int) -> None: if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=2 + ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. @@ -189,31 +195,20 @@ class Pybind11Extension(_Extension): current_macos = tuple(int(x) for x in platform.mac_ver()[0].split(".")[:2]) desired_macos = (10, 9) if level < 17 else (10, 14) macos_string = ".".join(str(x) for x in min(current_macos, desired_macos)) - macosx_min = "-mmacosx-version-min=" + macos_string + macosx_min = f"-mmacosx-version-min={macos_string}" cflags += [macosx_min] ldflags += [macosx_min] - if PY2: - if WIN: - # Will be ignored on MSVC 2015, where C++17 is not supported so - # this flag is not valid. - cflags += ["/wd5033"] - elif level >= 17: - cflags += ["-Wno-register"] - elif level >= 14: - cflags += ["-Wno-deprecated-register"] - self._add_cflags(cflags) self._add_ldflags(ldflags) # Just in case someone clever tries to multithread tmp_chdir_lock = threading.Lock() -cpp_cache_lock = threading.Lock() @contextlib.contextmanager -def tmp_chdir(): +def tmp_chdir() -> Iterator[str]: "Prepare and enter a temporary directory, cleanup when done" # Threadsafe @@ -229,7 +224,7 @@ def tmp_chdir(): # cf http://bugs.python.org/issue26689 -def has_flag(compiler, flag): +def has_flag(compiler: Any, flag: str) -> bool: """ Return the flag if a flag name is supported on the specified compiler, otherwise None (can be used as a boolean). @@ -237,13 +232,12 @@ def has_flag(compiler, flag): """ with tmp_chdir(): - fname = "flagcheck.cpp" - with open(fname, "w") as f: - # Don't trigger -Wunused-parameter. - f.write("int main (int, char **) { return 0; }") + fname = Path("flagcheck.cpp") + # Don't trigger -Wunused-parameter. + fname.write_text("int main (int, char **) { return 0; }", encoding="utf-8") try: - compiler.compile([fname], extra_postargs=[flag]) + compiler.compile([str(fname)], extra_postargs=[flag]) except distutils.errors.CompileError: return False return True @@ -253,7 +247,8 @@ def has_flag(compiler, flag): cpp_flag_cache = None -def auto_cpp_level(compiler): +@lru_cache() +def auto_cpp_level(compiler: Any) -> Union[str, int]: """ Return the max supported C++ std level (17, 14, or 11). Returns latest on Windows. """ @@ -261,19 +256,10 @@ def auto_cpp_level(compiler): if WIN: return "latest" - global cpp_flag_cache - - # If this has been previously calculated with the same args, return that - with cpp_cache_lock: - if cpp_flag_cache: - return cpp_flag_cache - levels = [17, 14, 11] for level in levels: if has_flag(compiler, STD_TMPL.format(level)): - with cpp_cache_lock: - cpp_flag_cache = level return level msg = "Unsupported compiler -- at least C++11 support is needed!" @@ -287,22 +273,61 @@ class build_ext(_build_ext): # noqa: N801 for now, and is completely optional otherwise. """ - def build_extensions(self): + def build_extensions(self) -> None: """ Build extensions, injecting C++ std for Pybind11Extension if needed. """ for ext in self.extensions: if hasattr(ext, "_cxx_level") and ext._cxx_level == 0: - # Python 2 syntax - old-style distutils class - ext.__class__.cxx_std.__set__(ext, auto_cpp_level(self.compiler)) + ext.cxx_std = auto_cpp_level(self.compiler) + + super().build_extensions() + + +def intree_extensions( + paths: Iterable[str], package_dir: Optional[Dict[str, str]] = None +) -> List[Pybind11Extension]: + """ + Generate Pybind11Extensions from source files directly located in a Python + source tree. + + ``package_dir`` behaves as in ``setuptools.setup``. If unset, the Python + package root parent is determined as the first parent directory that does + not contain an ``__init__.py`` file. + """ + exts = [] + + if package_dir is None: + for path in paths: + parent, _ = os.path.split(path) + while os.path.exists(os.path.join(parent, "__init__.py")): + parent, _ = os.path.split(parent) + relname, _ = os.path.splitext(os.path.relpath(path, parent)) + qualified_name = relname.replace(os.path.sep, ".") + exts.append(Pybind11Extension(qualified_name, [path])) + return exts + + for path in paths: + for prefix, parent in package_dir.items(): + if path.startswith(parent): + relname, _ = os.path.splitext(os.path.relpath(path, parent)) + qualified_name = relname.replace(os.path.sep, ".") + if prefix: + qualified_name = prefix + "." + qualified_name + exts.append(Pybind11Extension(qualified_name, [path])) + break + else: + msg = ( + f"path {path} is not a child of any of the directories listed " + f"in 'package_dir' ({package_dir})" + ) + raise ValueError(msg) - # Python 2 doesn't allow super here, since distutils uses old-style - # classes! - _build_ext.build_extensions(self) + return exts -def naive_recompile(obj, src): +def naive_recompile(obj: str, src: str) -> bool: """ This will recompile only if the source file changes. It does not check header files, so a more advanced function or Ccache is better if you have @@ -311,7 +336,7 @@ def naive_recompile(obj, src): return os.stat(obj).st_mtime < os.stat(src).st_mtime -def no_recompile(obg, src): +def no_recompile(obg: str, src: str) -> bool: # noqa: ARG001 """ This is the safest but slowest choice (and is the default) - will always recompile sources. @@ -319,15 +344,33 @@ def no_recompile(obg, src): return True +S = TypeVar("S", bound="ParallelCompile") + +CCompilerMethod = Callable[ + [ + distutils.ccompiler.CCompiler, + List[str], + Optional[str], + Optional[Union[Tuple[str], Tuple[str, Optional[str]]]], + Optional[List[str]], + bool, + Optional[List[str]], + Optional[List[str]], + Optional[List[str]], + ], + List[str], +] + + # Optional parallel compile utility # inspired by: http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils # and: https://github.com/tbenthompson/cppimport/blob/stable/cppimport/build_module.py # and NumPy's parallel distutils module: # https://github.com/numpy/numpy/blob/master/numpy/distutils/ccompiler.py -class ParallelCompile(object): +class ParallelCompile: """ Make a parallel compile function. Inspired by - numpy.distutils.ccompiler.CCompiler_compile and cppimport. + numpy.distutils.ccompiler.CCompiler.compile and cppimport. This takes several arguments that allow you to customize the compile function created: @@ -362,35 +405,40 @@ class ParallelCompile(object): __slots__ = ("envvar", "default", "max", "_old", "needs_recompile") - def __init__(self, envvar=None, default=0, max=0, needs_recompile=no_recompile): + def __init__( + self, + envvar: Optional[str] = None, + default: int = 0, + max: int = 0, # pylint: disable=redefined-builtin + needs_recompile: Callable[[str, str], bool] = no_recompile, + ) -> None: self.envvar = envvar self.default = default self.max = max self.needs_recompile = needs_recompile - self._old = [] + self._old: List[CCompilerMethod] = [] - def function(self): + def function(self) -> CCompilerMethod: """ Builds a function object usable as distutils.ccompiler.CCompiler.compile. """ def compile_function( - compiler, - sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None, - ): - + compiler: distutils.ccompiler.CCompiler, + sources: List[str], + output_dir: Optional[str] = None, + macros: Optional[Union[Tuple[str], Tuple[str, Optional[str]]]] = None, + include_dirs: Optional[List[str]] = None, + debug: bool = False, + extra_preargs: Optional[List[str]] = None, + extra_postargs: Optional[List[str]] = None, + depends: Optional[List[str]] = None, + ) -> Any: # These lines are directly from distutils.ccompiler.CCompiler - macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( + macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs ) - cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) + cc_args = compiler._get_cc_args(pp_opts, debug, extra_preargs) # type: ignore[attr-defined] # The number of threads; start with default. threads = self.default @@ -399,17 +447,19 @@ class ParallelCompile(object): if self.envvar is not None: threads = int(os.environ.get(self.envvar, self.default)) - def _single_compile(obj): + def _single_compile(obj: Any) -> None: try: src, ext = build[obj] except KeyError: return if not os.path.exists(obj) or self.needs_recompile(obj, src): - compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + compiler._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) # type: ignore[attr-defined] try: - import multiprocessing + # Importing .synchronize checks for platforms that have some multiprocessing + # capabilities but lack semaphores, such as AWS Lambda and Android Termux. + import multiprocessing.synchronize from multiprocessing.pool import ThreadPool except ImportError: threads = 1 @@ -422,8 +472,9 @@ class ParallelCompile(object): threads = 1 if threads > 1: - for _ in ThreadPool(threads).imap_unordered(_single_compile, objects): - pass + with ThreadPool(threads) as pool: + for _ in pool.imap_unordered(_single_compile, objects): + pass else: for ob in objects: _single_compile(ob) @@ -432,13 +483,16 @@ class ParallelCompile(object): return compile_function - def install(self): - distutils.ccompiler.CCompiler.compile = self.function() + def install(self: S) -> S: + """ + Installs the compile function into distutils.ccompiler.CCompiler.compile. + """ + distutils.ccompiler.CCompiler.compile = self.function() # type: ignore[assignment] return self - def __enter__(self): + def __enter__(self: S) -> S: self._old.append(distutils.ccompiler.CCompiler.compile) return self.install() - def __exit__(self, *args): - distutils.ccompiler.CCompiler.compile = self._old.pop() + def __exit__(self, *args: Any) -> None: + distutils.ccompiler.CCompiler.compile = self._old.pop() # type: ignore[assignment] diff --git a/pybind11/setup_helpers.pyi b/pybind11/setup_helpers.pyi deleted file mode 100644 index 23232e1f..00000000 --- a/pybind11/setup_helpers.pyi +++ /dev/null @@ -1,61 +0,0 @@ -# IMPORTANT: Should stay in sync with setup_helpers.py (mostly checked by CI / -# pre-commit). - -from typing import Any, Callable, Iterator, Optional, Type, TypeVar, Union -from types import TracebackType - -from distutils.command.build_ext import build_ext as _build_ext # type: ignore -from distutils.extension import Extension as _Extension -import distutils.ccompiler -import contextlib - -WIN: bool -PY2: bool -MACOS: bool -STD_TMPL: str - -class Pybind11Extension(_Extension): - def _add_cflags(self, *flags: str) -> None: ... - def _add_lflags(self, *flags: str) -> None: ... - def __init__( - self, *args: Any, cxx_std: int = 0, language: str = "c++", **kwargs: Any - ) -> None: ... - @property - def cxx_std(self) -> int: ... - @cxx_std.setter - def cxx_std(self, level: int) -> None: ... - -@contextlib.contextmanager -def tmp_chdir() -> Iterator[str]: ... -def has_flag(compiler: distutils.ccompiler.CCompiler, flag: str) -> bool: ... -def auto_cpp_level(compiler: distutils.ccompiler.CCompiler) -> Union[int, str]: ... - -class build_ext(_build_ext): # type: ignore - def build_extensions(self) -> None: ... - -def no_recompile(obj: str, src: str) -> bool: ... -def naive_recompile(obj: str, src: str) -> bool: ... - -T = TypeVar("T", bound="ParallelCompile") - -class ParallelCompile: - envvar: Optional[str] - default: int - max: int - needs_recompile: Callable[[str, str], bool] - def __init__( - self, - envvar: Optional[str] = None, - default: int = 0, - max: int = 0, - needs_recompile: Callable[[str, str], bool] = no_recompile, - ) -> None: ... - def function(self) -> Any: ... - def install(self: T) -> T: ... - def __enter__(self: T) -> T: ... - def __exit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> None: ... diff --git a/pyproject.toml b/pyproject.toml index 5c9d1535..59c15ea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,98 @@ [build-system] -requires = ["setuptools>=42", "wheel", "cmake>=3.18", "ninja"] +requires = ["setuptools>=42", "cmake>=3.18", "ninja"] build-backend = "setuptools.build_meta" + + +[tool.check-manifest] +ignore = [ + "tests/**", + "docs/**", + "tools/**", + "include/**", + ".*", + "pybind11/include/**", + "pybind11/share/**", + "CMakeLists.txt", + "noxfile.py", +] + + +[tool.mypy] +files = ["pybind11"] +python_version = "3.6" +strict = true +show_error_codes = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] +warn_unreachable = true + +[[tool.mypy.overrides]] +module = ["ghapi.*"] +ignore_missing_imports = true + + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] +xfail_strict = true +filterwarnings = ["error"] +log_cli_level = "info" +testpaths = [ + "tests", +] +timeout=300 + + +[tool.pylint] +master.py-version = "3.6" +reports.output-format = "colorized" +messages_control.disable = [ + "design", + "fixme", + "imports", + "line-too-long", + "imports", + "invalid-name", + "protected-access", + "missing-module-docstring", + "unused-argument", # covered by Ruff ARG +] + + +[tool.ruff] +select = [ + "E", "F", "W", # flake8 + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF100", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 +] +ignore = [ + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118", # iter(x) is not always the same as iter(x.keys()) +] +target-version = "py37" +src = ["src"] +unfixable = ["T20"] +exclude = [] +line-length = 120 +isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] + +[tool.ruff.per-file-ignores] +"tests/**" = ["EM", "N"] +"tests/test_call_policies.py" = ["PLC1901"] @@ -13,13 +13,14 @@ classifiers = Topic :: Software Development :: Libraries :: Python Modules Topic :: Utilities Programming Language :: C++ - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython @@ -30,56 +31,13 @@ keywords = C++11 Python bindings +project_urls = + Documentation = https://pybind11.readthedocs.io/ + Bug Tracker = https://github.com/pybind/pybind11/issues + Discussions = https://github.com/pybind/pybind11/discussions + Changelog = https://pybind11.readthedocs.io/en/latest/changelog.html + Chat = https://gitter.im/pybind/Lobby + [options] -python_requires = >=2.7, !=3.0, !=3.1, !=3.2, !=3.3, !=3.4 +python_requires = >=3.6 zip_safe = False - -[bdist_wheel] -universal=1 - -[check-manifest] -ignore = - tests/** - docs/** - tools/** - include/** - .* - pybind11/include/** - pybind11/share/** - CMakeLists.txt - - -[flake8] -max-line-length = 99 -show_source = True -exclude = .git, __pycache__, build, dist, docs, tools, venv -ignore = - # required for pretty matrix formatting: multiple spaces after `,` and `[` - E201, E241, W504, - # camelcase 'cPickle' imported as lowercase 'pickle' - N813 - # Black conflict - W503, E203 - -[mypy] -files = pybind11 -python_version = 2.7 -warn_unused_configs = True - -# Currently (0.800) identical to --strict -disallow_any_generics = True -disallow_subclassing_any = True -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_redundant_casts = True -warn_unused_ignores = True -warn_return_any = True -no_implicit_reexport = True -strict_equality = True - -[tool:pytest] -timeout = 300 @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # Setup script for PyPI; use CMakeFile.txt to build extension modules @@ -10,87 +9,110 @@ import shutil import string import subprocess import sys -import tempfile +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Dict, Iterator, List, Union import setuptools.command.sdist -DIR = os.path.abspath(os.path.dirname(__file__)) +DIR = Path(__file__).parent.absolute() VERSION_REGEX = re.compile( r"^\s*#\s*define\s+PYBIND11_VERSION_([A-Z]+)\s+(.*)$", re.MULTILINE ) +VERSION_FILE = Path("pybind11/_version.py") +COMMON_FILE = Path("include/pybind11/detail/common.h") + + +def build_expected_version_hex(matches: Dict[str, str]) -> str: + patch_level_serial = matches["PATCH"] + serial = None + major = int(matches["MAJOR"]) + minor = int(matches["MINOR"]) + flds = patch_level_serial.split(".") + if flds: + patch = int(flds[0]) + if len(flds) == 1: + level = "0" + serial = 0 + elif len(flds) == 2: + level_serial = flds[1] + for level in ("a", "b", "c", "dev"): + if level_serial.startswith(level): + serial = int(level_serial[len(level) :]) + break + if serial is None: + msg = f'Invalid PYBIND11_VERSION_PATCH: "{patch_level_serial}"' + raise RuntimeError(msg) + version_hex_str = f"{major:02x}{minor:02x}{patch:02x}{level[:1]}{serial:x}" + return f"0x{version_hex_str.upper()}" + # PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers # files, and the sys.prefix files (CMake and headers). global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False) -setup_py = "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" +setup_py = Path( + "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in" +) extra_cmd = 'cmdclass["sdist"] = SDist\n' to_src = ( - ("pyproject.toml", "tools/pyproject.toml"), - ("setup.py", setup_py), + (Path("pyproject.toml"), Path("tools/pyproject.toml")), + (Path("setup.py"), setup_py), ) + # Read the listed version -with open("pybind11/_version.py") as f: - code = compile(f.read(), "pybind11/_version.py", "exec") -loc = {} +loc: Dict[str, str] = {} +code = compile(VERSION_FILE.read_text(encoding="utf-8"), "pybind11/_version.py", "exec") exec(code, loc) version = loc["__version__"] # Verify that the version matches the one in C++ -with open("include/pybind11/detail/common.h") as f: - matches = dict(VERSION_REGEX.findall(f.read())) +matches = dict(VERSION_REGEX.findall(COMMON_FILE.read_text(encoding="utf8"))) cpp_version = "{MAJOR}.{MINOR}.{PATCH}".format(**matches) if version != cpp_version: - msg = "Python version {} does not match C++ version {}!".format( - version, cpp_version - ) + msg = f"Python version {version} does not match C++ version {cpp_version}!" + raise RuntimeError(msg) + +version_hex = matches.get("HEX", "MISSING") +exp_version_hex = build_expected_version_hex(matches) +if version_hex != exp_version_hex: + msg = f"PYBIND11_VERSION_HEX {version_hex} does not match expected value {exp_version_hex}!" raise RuntimeError(msg) -def get_and_replace(filename, binary=False, **opts): - with open(filename, "rb" if binary else "r") as f: - contents = f.read() - # Replacement has to be done on text in Python 3 (both work in Python 2) +# TODO: use literals & overload (typing extensions or Python 3.8) +def get_and_replace( + filename: Path, binary: bool = False, **opts: str +) -> Union[bytes, str]: if binary: + contents = filename.read_bytes() return string.Template(contents.decode()).substitute(opts).encode() - else: - return string.Template(contents).substitute(opts) + + return string.Template(filename.read_text()).substitute(opts) # Use our input files instead when making the SDist (and anything that depends # on it, like a wheel) class SDist(setuptools.command.sdist.sdist): - def make_release_tree(self, base_dir, files): - setuptools.command.sdist.sdist.make_release_tree(self, base_dir, files) + def make_release_tree(self, base_dir: str, files: List[str]) -> None: + super().make_release_tree(base_dir, files) for to, src in to_src: txt = get_and_replace(src, binary=True, version=version, extra_cmd="") - dest = os.path.join(base_dir, to) + dest = Path(base_dir) / to # This is normally linked, so unlink before writing! - os.unlink(dest) - with open(dest, "wb") as f: - f.write(txt) - - -# Backport from Python 3 -@contextlib.contextmanager -def TemporaryDirectory(): # noqa: N802 - "Prepare a temporary directory, cleanup when done" - try: - tmpdir = tempfile.mkdtemp() - yield tmpdir - finally: - shutil.rmtree(tmpdir) + dest.unlink() + dest.write_bytes(txt) # type: ignore[arg-type] # Remove the CMake install directory when done @contextlib.contextmanager -def remove_output(*sources): +def remove_output(*sources: str) -> Iterator[None]: try: yield finally: @@ -105,10 +127,23 @@ with remove_output("pybind11/include", "pybind11/share"): "-DCMAKE_INSTALL_PREFIX=pybind11", "-DBUILD_TESTING=OFF", "-DPYBIND11_NOPYTHON=ON", + "-Dprefix_for_pc_file=${pcfiledir}/../../", ] - cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) - subprocess.check_call(cmd, **cmake_opts) - subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts) + if "CMAKE_ARGS" in os.environ: + fcommand = [ + c + for c in os.environ["CMAKE_ARGS"].split() + if "DCMAKE_INSTALL_PREFIX" not in c + ] + cmd += fcommand + subprocess.run(cmd, check=True, cwd=DIR, stdout=sys.stdout, stderr=sys.stderr) + subprocess.run( + ["cmake", "--install", tmpdir], + check=True, + cwd=DIR, + stdout=sys.stdout, + stderr=sys.stderr, + ) txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd) code = compile(txt, setup_py, "exec") diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3bfd5f1c..80ee9c1f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,21 +5,18 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() -# Only needed for CMake < 3.5 support -include(CMakeParseArguments) - -# Filter out items; print an optional message if any items filtered +# Filter out items; print an optional message if any items filtered. This ignores extensions. # # Usage: # pybind11_filter_tests(LISTNAME file1.cpp file2.cpp ... MESSAGE "") @@ -27,10 +24,17 @@ include(CMakeParseArguments) macro(pybind11_filter_tests LISTNAME) cmake_parse_arguments(ARG "" "MESSAGE" "" ${ARGN}) set(PYBIND11_FILTER_TESTS_FOUND OFF) + # Make a list of the test without any extensions, for easier filtering. + set(_TMP_ACTUAL_LIST "${${LISTNAME}};") # enforce ';' at the end to allow matching last item. + string(REGEX REPLACE "\\.[^.;]*;" ";" LIST_WITHOUT_EXTENSIONS "${_TMP_ACTUAL_LIST}") foreach(filename IN LISTS ARG_UNPARSED_ARGUMENTS) - list(FIND ${LISTNAME} ${filename} _FILE_FOUND) + string(REGEX REPLACE "\\.[^.]*$" "" filename_no_ext ${filename}) + # Search in the list without extensions. + list(FIND LIST_WITHOUT_EXTENSIONS ${filename_no_ext} _FILE_FOUND) if(_FILE_FOUND GREATER -1) - list(REMOVE_AT ${LISTNAME} ${_FILE_FOUND}) + list(REMOVE_AT ${LISTNAME} ${_FILE_FOUND}) # And remove from the list with extensions. + list(REMOVE_AT LIST_WITHOUT_EXTENSIONS ${_FILE_FOUND} + )# And our search list, to ensure it is in sync. set(PYBIND11_FILTER_TESTS_FOUND ON) endif() endforeach() @@ -47,6 +51,18 @@ macro(possibly_uninitialized) endforeach() endmacro() +# Function to add additional targets if any of the provided tests are found. +# Needles; Specifies the test names to look for. +# Additions; Specifies the additional test targets to add when any of the needles are found. +macro(tests_extra_targets needles additions) + # Add the index for this relation to the index extra targets map. + list(LENGTH PYBIND11_TEST_EXTRA_TARGETS PYBIND11_TEST_EXTRA_TARGETS_LEN) + list(APPEND PYBIND11_TEST_EXTRA_TARGETS ${PYBIND11_TEST_EXTRA_TARGETS_LEN}) + # Add the test names to look for, and the associated test target additions. + set(PYBIND11_TEST_EXTRA_TARGETS_NEEDLES_${PYBIND11_TEST_EXTRA_TARGETS_LEN} ${needles}) + set(PYBIND11_TEST_EXTRA_TARGETS_ADDITION_${PYBIND11_TEST_EXTRA_TARGETS_LEN} ${additions}) +endmacro() + # New Python support if(DEFINED Python_EXECUTABLE) set(PYTHON_EXECUTABLE "${Python_EXECUTABLE}") @@ -92,52 +108,72 @@ if(PYBIND11_CUDA_TESTS) set(CMAKE_CUDA_STANDARD_REQUIRED ON) endif() -# Full set of test files (you can override these; see below) +# Full set of test files (you can override these; see below, overrides ignore extension) +# Any test that has no extension is both .py and .cpp, so 'foo' will add 'foo.cpp' and 'foo.py'. +# Any test that has an extension is exclusively that and handled as such. set(PYBIND11_TEST_FILES - test_async.cpp - test_buffers.cpp - test_builtin_casters.cpp - test_call_policies.cpp - test_callbacks.cpp - test_chrono.cpp - test_class.cpp - test_constants_and_functions.cpp - test_copy_move.cpp - test_custom_type_casters.cpp - test_docstring_options.cpp - test_eigen.cpp - test_enum.cpp - test_eval.cpp - test_exceptions.cpp - test_factory_constructors.cpp - test_gil_scoped.cpp - test_iostream.cpp - test_kwargs_and_defaults.cpp - test_local_bindings.cpp - test_methods_and_attributes.cpp - test_modules.cpp - test_multiple_inheritance.cpp - test_numpy_array.cpp - test_numpy_dtypes.cpp - test_numpy_vectorize.cpp - test_opaque_types.cpp - test_operator_overloading.cpp - test_pickling.cpp - test_pytypes.cpp - test_sequences_and_iterators.cpp - test_smart_ptr.cpp - test_stl.cpp - test_stl_binders.cpp - test_tagbased_polymorphic.cpp - test_union.cpp - test_virtual_functions.cpp) + test_async + test_buffers + test_builtin_casters + test_call_policies + test_callbacks + test_chrono + test_class + test_const_name + test_constants_and_functions + test_copy_move + test_custom_type_casters + test_custom_type_setup + test_docstring_options + test_eigen_matrix + test_eigen_tensor + test_enum + test_eval + test_exceptions + test_factory_constructors + test_gil_scoped + test_iostream + test_kwargs_and_defaults + test_local_bindings + test_methods_and_attributes + test_modules + test_multiple_inheritance + test_numpy_array + test_numpy_dtypes + test_numpy_vectorize + test_opaque_types + test_operator_overloading + test_pickling + test_pytypes + test_sequences_and_iterators + test_smart_ptr + test_stl + test_stl_binders + test_tagbased_polymorphic + test_thread + test_type_caster_pyobject_ptr + test_union + test_unnamed_namespace_a + test_unnamed_namespace_b + test_vector_unique_ptr_member + test_virtual_functions) # Invoking cmake with something like: # cmake -DPYBIND11_TEST_OVERRIDE="test_callbacks.cpp;test_pickling.cpp" .. # lets you override the tests that get compiled and run. You can restore to all tests with: # cmake -DPYBIND11_TEST_OVERRIDE= .. if(PYBIND11_TEST_OVERRIDE) - set(PYBIND11_TEST_FILES ${PYBIND11_TEST_OVERRIDE}) + # Instead of doing a direct override here, we iterate over the overrides without extension and + # match them against entries from the PYBIND11_TEST_FILES, anything that not matches goes into the filter list. + string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_OVERRIDE_NO_EXT "${PYBIND11_TEST_OVERRIDE};") + string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_FILES_NO_EXT "${PYBIND11_TEST_FILES};") + # This allows the override to be done with extensions, preserving backwards compatibility. + foreach(test_name ${TEST_FILES_NO_EXT}) + if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT + )# If not in the allowlist, add to be filtered out. + list(APPEND PYBIND11_TEST_FILTER ${test_name}) + endif() + endforeach() endif() # You can also filter tests: @@ -145,11 +181,6 @@ if(PYBIND11_TEST_FILTER) pybind11_filter_tests(PYBIND11_TEST_FILES ${PYBIND11_TEST_FILTER}) endif() -if(PYTHON_VERSION VERSION_LESS 3.5) - pybind11_filter_tests(PYBIND11_TEST_FILES test_async.cpp MESSAGE - "Skipping test_async on Python 2") -endif() - # Skip tests for CUDA check: # /pybind11/tests/test_constants_and_functions.cpp(125): # error: incompatible exception specifications @@ -159,20 +190,55 @@ if(PYBIND11_CUDA_TESTS) "Skipping test_constants_and_functions due to incompatible exception specifications") endif() -string(REPLACE ".cpp" ".py" PYBIND11_PYTEST_FILES "${PYBIND11_TEST_FILES}") +# Now that the test filtering is complete, we need to split the list into the test for PYTEST +# and the list for the cpp targets. +set(PYBIND11_CPPTEST_FILES "") +set(PYBIND11_PYTEST_FILES "") + +foreach(test_name ${PYBIND11_TEST_FILES}) + if(test_name MATCHES "\\.py$") # Ends in .py, purely python test. + list(APPEND PYBIND11_PYTEST_FILES ${test_name}) + elseif(test_name MATCHES "\\.cpp$") # Ends in .cpp, purely cpp test. + list(APPEND PYBIND11_CPPTEST_FILES ${test_name}) + elseif(NOT test_name MATCHES "\\.") # No extension specified, assume both, add extension. + list(APPEND PYBIND11_PYTEST_FILES ${test_name}.py) + list(APPEND PYBIND11_CPPTEST_FILES ${test_name}.cpp) + else() + message(WARNING "Unhanded test extension in test: ${test_name}") + endif() +endforeach() +set(PYBIND11_TEST_FILES ${PYBIND11_CPPTEST_FILES}) +list(SORT PYBIND11_PYTEST_FILES) # Contains the set of test files that require pybind11_cross_module_tests to be # built; if none of these are built (i.e. because TEST_OVERRIDE is used and # doesn't include them) the second module doesn't get built. -set(PYBIND11_CROSS_MODULE_TESTS test_exceptions.py test_local_bindings.py test_stl.py - test_stl_binders.py) +tests_extra_targets("test_exceptions.py;test_local_bindings.py;test_stl.py;test_stl_binders.py" + "pybind11_cross_module_tests") + +# And add additional targets for other tests. +tests_extra_targets("test_exceptions.py" "cross_module_interleaved_error_already_set") +tests_extra_targets("test_gil_scoped.py" "cross_module_gil_utils") -set(PYBIND11_CROSS_MODULE_GIL_TESTS test_gil_scoped.py) +set(PYBIND11_EIGEN_REPO + "https://gitlab.com/libeigen/eigen.git" + CACHE STRING "Eigen repository to use for tests") +# Always use a hash for reconfigure speed and security reasons +# Include the version number for pretty printing (keep in sync) +set(PYBIND11_EIGEN_VERSION_AND_HASH + "3.4.0;929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec" + CACHE STRING "Eigen version to use for tests, format: VERSION;HASH") + +list(GET PYBIND11_EIGEN_VERSION_AND_HASH 0 PYBIND11_EIGEN_VERSION_STRING) +list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH) # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed" # skip message). -list(FIND PYBIND11_TEST_FILES test_eigen.cpp PYBIND11_TEST_FILES_EIGEN_I) +list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) +if(PYBIND11_TEST_FILES_EIGEN_I EQUAL -1) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) +endif() if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake). # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also @@ -182,22 +248,26 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) message(FATAL_ERROR "CMake 3.11+ required when using DOWNLOAD_EIGEN") endif() - set(EIGEN3_VERSION_STRING "3.3.8") - include(FetchContent) FetchContent_Declare( eigen - GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git - GIT_TAG ${EIGEN3_VERSION_STRING}) + GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}" + GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}") FetchContent_GetProperties(eigen) if(NOT eigen_POPULATED) - message(STATUS "Downloading Eigen") + message( + STATUS + "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}" + ) FetchContent_Populate(eigen) endif() set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR}) set(EIGEN3_FOUND TRUE) + # When getting locally, the version is not visible from a superprojet, + # so just force it. + set(EIGEN3_VERSION "${PYBIND11_EIGEN_VERSION_STRING}") else() find_package(Eigen3 3.2.7 QUIET CONFIG) @@ -223,9 +293,31 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) set(EIGEN3_VERSION ${EIGEN3_VERSION_STRING}) endif() message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") + + if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) + tests_extra_targets("test_eigen_tensor.py" "eigen_tensor_avoid_stl_array") + endif() + else() + list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() + + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() + message( + STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") + endif() +endif() + +# Some code doesn't support gcc 4 +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) - message(STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN on CMake 3.11+ to download") endif() endif() @@ -246,10 +338,47 @@ if(Boost_FOUND) endif() endif() +# Check if we need to add -lstdc++fs or -lc++fs or nothing +if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD LESS 17) + set(STD_FS_NO_LIB_NEEDED TRUE) +elseif(MSVC) + set(STD_FS_NO_LIB_NEEDED TRUE) +else() + file( + WRITE ${CMAKE_CURRENT_BINARY_DIR}/main.cpp + "#include <filesystem>\nint main(int argc, char ** argv) {\n std::filesystem::path p(argv[0]);\n return p.string().length();\n}" + ) + try_compile( + STD_FS_NO_LIB_NEEDED ${CMAKE_CURRENT_BINARY_DIR} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/main.cpp + COMPILE_DEFINITIONS -std=c++17) + try_compile( + STD_FS_NEEDS_STDCXXFS ${CMAKE_CURRENT_BINARY_DIR} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/main.cpp + COMPILE_DEFINITIONS -std=c++17 + LINK_LIBRARIES stdc++fs) + try_compile( + STD_FS_NEEDS_CXXFS ${CMAKE_CURRENT_BINARY_DIR} + SOURCES ${CMAKE_CURRENT_BINARY_DIR}/main.cpp + COMPILE_DEFINITIONS -std=c++17 + LINK_LIBRARIES c++fs) +endif() + +if(${STD_FS_NEEDS_STDCXXFS}) + set(STD_FS_LIB stdc++fs) +elseif(${STD_FS_NEEDS_CXXFS}) + set(STD_FS_LIB c++fs) +elseif(${STD_FS_NO_LIB_NEEDED}) + set(STD_FS_LIB "") +else() + message(WARNING "Unknown C++17 compiler - not passing -lstdc++fs") + set(STD_FS_LIB "") +endif() + # Compile with compiler warnings turned on function(pybind11_enable_warnings target_name) if(MSVC) - target_compile_options(${target_name} PRIVATE /W4) + target_compile_options(${target_name} PRIVATE /W4 /wd4189) elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)" AND NOT PYBIND11_CUDA_TESTS) target_compile_options( ${target_name} @@ -267,40 +396,35 @@ function(pybind11_enable_warnings target_name) target_compile_options(${target_name} PRIVATE /WX) elseif(PYBIND11_CUDA_TESTS) target_compile_options(${target_name} PRIVATE "SHELL:-Werror all-warnings") - elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Intel|Clang)") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Clang|IntelLLVM)") target_compile_options(${target_name} PRIVATE -Werror) - endif() - endif() - - # Needs to be readded since the ordering requires these to be after the ones above - if(CMAKE_CXX_STANDARD - AND CMAKE_CXX_COMPILER_ID MATCHES "Clang" - AND PYTHON_VERSION VERSION_LESS 3.0) - if(CMAKE_CXX_STANDARD LESS 17) - target_compile_options(${target_name} PUBLIC -Wno-deprecated-register) - else() - target_compile_options(${target_name} PUBLIC -Wno-register) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if(CMAKE_CXX_STANDARD EQUAL 17) # See PR #3570 + target_compile_options(${target_name} PRIVATE -Wno-conversion) + endif() + target_compile_options( + ${target_name} + PRIVATE + -Werror-all + # "Inlining inhibited by limit max-size", "Inlining inhibited by limit max-total-size" + -diag-disable 11074,11076) endif() endif() endfunction() set(test_targets pybind11_tests) -# Build pybind11_cross_module_tests if any test_whatever.py are being built that require it -foreach(t ${PYBIND11_CROSS_MODULE_TESTS}) - list(FIND PYBIND11_PYTEST_FILES ${t} i) - if(i GREATER -1) - list(APPEND test_targets pybind11_cross_module_tests) - break() - endif() -endforeach() - -foreach(t ${PYBIND11_CROSS_MODULE_GIL_TESTS}) - list(FIND PYBIND11_PYTEST_FILES ${t} i) - if(i GREATER -1) - list(APPEND test_targets cross_module_gil_utils) - break() - endif() +# Check if any tests need extra targets by iterating through the mappings registered. +foreach(i ${PYBIND11_TEST_EXTRA_TARGETS}) + foreach(needle ${PYBIND11_TEST_EXTRA_TARGETS_NEEDLES_${i}}) + if(needle IN_LIST PYBIND11_PYTEST_FILES) + # Add all the additional targets to the test list. List join in newer cmake. + foreach(extra_target ${PYBIND11_TEST_EXTRA_TARGETS_ADDITION_${i}}) + list(APPEND test_targets ${extra_target}) + endforeach() + break() # Breaks out of the needle search, continues with the next mapping. + endif() + endforeach() endforeach() # Support CUDA testing by forcing the target file to compile with NVCC @@ -349,6 +473,8 @@ foreach(target ${test_targets}) target_compile_definitions(${target} PRIVATE -DPYBIND11_TEST_BOOST) endif() + target_link_libraries(${target} PRIVATE ${STD_FS_LIB}) + # Always write the output file directly into the 'tests' directory (even on MSVC) if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY) set_target_properties(${target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY @@ -364,6 +490,14 @@ foreach(target ${test_targets}) endif() endforeach() +# Provide nice organisation in IDEs +if(NOT CMAKE_VERSION VERSION_LESS 3.8) + source_group( + TREE "${CMAKE_CURRENT_SOURCE_DIR}/../include" + PREFIX "Header Files" + FILES ${PYBIND11_HEADERS}) +endif() + # Make sure pytest is found or produce a warning pybind11_find_import(pytest VERSION 3.1) diff --git a/tests/conftest.py b/tests/conftest.py index 362eb806..ad5b47b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,31 +1,48 @@ -# -*- coding: utf-8 -*- """pytest configuration Extends output capture as needed by pybind11: ignore constructors, optional unordered lines. -Adds docstring and exceptions message sanitizers: ignore Python 2 vs 3 differences. +Adds docstring and exceptions message sanitizers. """ import contextlib import difflib import gc +import multiprocessing import re +import sys import textwrap +import traceback import pytest -import env - # Early diagnostic for failed imports -import pybind11_tests # noqa: F401 +try: + import pybind11_tests +except Exception: + # pytest does not show the traceback without this. + traceback.print_exc() + raise + + +@pytest.fixture(scope="session", autouse=True) +def use_multiprocessing_forkserver_on_linux(): + if sys.platform != "linux": + # The default on Windows and macOS is "spawn": If it's not broken, don't fix it. + return + + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 + # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. + # It is actually a well-known pitfall, unfortunately without guard rails. + # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, + # visit the issuecomment link above for details). + multiprocessing.set_start_method("forkserver") + -_unicode_marker = re.compile(r"u(\'[^\']*\')") _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") # Avoid collecting Python3 only files collect_ignore = [] -if env.PY2: - collect_ignore.append("test_async.py") def _strip_and_dedent(s): @@ -45,7 +62,7 @@ def _make_explanation(a, b): ] -class Output(object): +class Output: """Basic output post-processing and comparison""" def __init__(self, string): @@ -65,9 +82,8 @@ class Output(object): b = _strip_and_dedent(other).splitlines() if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Unordered(Output): @@ -78,12 +94,11 @@ class Unordered(Output): b = _split_and_sort(other) if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False -class Capture(object): +class Capture: def __init__(self, capfd): self.capfd = capfd self.out = "" @@ -101,9 +116,8 @@ class Capture(object): b = other if a == b: return True - else: - self.explanation = a.explanation - return False + self.explanation = a.explanation + return False def __str__(self): return self.out @@ -120,13 +134,13 @@ class Capture(object): return Output(self.err) -@pytest.fixture +@pytest.fixture() def capture(capsys): """Extended `capsys` with context manager and custom equality operators""" return Capture(capsys) -class SanitizedString(object): +class SanitizedString: def __init__(self, sanitizer): self.sanitizer = sanitizer self.string = "" @@ -141,27 +155,22 @@ class SanitizedString(object): b = _strip_and_dedent(other) if a == b: return True - else: - self.explanation = _make_explanation(a.splitlines(), b.splitlines()) - return False + self.explanation = _make_explanation(a.splitlines(), b.splitlines()) + return False def _sanitize_general(s): s = s.strip() s = s.replace("pybind11_tests.", "m.") - s = s.replace("unicode", "str") - s = _long_marker.sub(r"\1", s) - s = _unicode_marker.sub(r"\1", s) - return s + return _long_marker.sub(r"\1", s) def _sanitize_docstring(thing): s = thing.__doc__ - s = _sanitize_general(s) - return s + return _sanitize_general(s) -@pytest.fixture +@pytest.fixture() def doc(): """Sanitize docstrings and add custom failure explanation""" return SanitizedString(_sanitize_docstring) @@ -170,30 +179,20 @@ def doc(): def _sanitize_message(thing): s = str(thing) s = _sanitize_general(s) - s = _hexadecimal.sub("0", s) - return s + return _hexadecimal.sub("0", s) -@pytest.fixture +@pytest.fixture() def msg(): """Sanitize messages and add custom failure explanation""" return SanitizedString(_sanitize_message) -# noinspection PyUnusedLocal -def pytest_assertrepr_compare(op, left, right): +def pytest_assertrepr_compare(op, left, right): # noqa: ARG001 """Hook to insert custom failure explanation""" if hasattr(left, "explanation"): return left.explanation - - -@contextlib.contextmanager -def suppress(exception): - """Suppress the desired exception""" - try: - yield - except exception: - pass + return None def gc_collect(): @@ -204,5 +203,19 @@ def gc_collect(): def pytest_configure(): - pytest.suppress = suppress + pytest.suppress = contextlib.suppress pytest.gc_collect = gc_collect + + +def pytest_report_header(config): + del config # Unused. + assert ( + pybind11_tests.compiler_info is not None + ), "Please update pybind11_tests.cpp if this assert fails." + return ( + "C++ Info:" + f" {pybind11_tests.compiler_info}" + f" {pybind11_tests.cpp_std}" + f" {pybind11_tests.PYBIND11_INTERNALS_ID}" + f" PYBIND11_SIMPLE_GIL_MANAGEMENT={pybind11_tests.PYBIND11_SIMPLE_GIL_MANAGEMENT}" + ) diff --git a/tests/constructor_stats.h b/tests/constructor_stats.h index 805968a0..937f6c23 100644 --- a/tests/constructor_stats.h +++ b/tests/constructor_stats.h @@ -56,7 +56,8 @@ from the ConstructorStats instance `.values()` method. In some cases, when you need to track instances of a C++ class not registered with pybind11, you need to add a function returning the ConstructorStats for the C++ class; this can be done with: - m.def("get_special_cstats", &ConstructorStats::get<SpecialClass>, py::return_value_policy::reference) + m.def("get_special_cstats", &ConstructorStats::get<SpecialClass>, +py::return_value_policy::reference) Finally, you can suppress the output messages, but keep the constructor tracking (for inspection/testing in python) by using the functions with `print_` replaced with `track_` (e.g. @@ -65,15 +66,18 @@ inspection/testing in python) by using the functions with `print_` replaced with */ #include "pybind11_tests.h" -#include <unordered_map> + #include <list> -#include <typeindex> #include <sstream> +#include <typeindex> +#include <unordered_map> class ConstructorStats { protected: - std::unordered_map<void*, int> _instances; // Need a map rather than set because members can shared address with parents - std::list<std::string> _values; // Used to track values (e.g. of value constructors) + std::unordered_map<void *, int> _instances; // Need a map rather than set because members can + // shared address with parents + std::list<std::string> _values; // Used to track values + // (e.g. of value constructors) public: int default_constructions = 0; int copy_constructions = 0; @@ -96,26 +100,26 @@ public: default_constructions++; } - void created(void *inst) { - ++_instances[inst]; - } + void created(void *inst) { ++_instances[inst]; } void destroyed(void *inst) { - if (--_instances[inst] < 0) + if (--_instances[inst] < 0) { throw std::runtime_error("cstats.destroyed() called with unknown " "instance; potential double-destruction " "or a missing cstats.created()"); + } } static void gc() { // Force garbage collection to ensure any pending destructors are invoked: #if defined(PYPY_VERSION) PyObject *globals = PyEval_GetGlobals(); - PyObject *result = PyRun_String( - "import gc\n" - "for i in range(2):" - " gc.collect()\n", - Py_file_input, globals, globals); + PyObject *result = PyRun_String("import gc\n" + "for i in range(2):\n" + " gc.collect()\n", + Py_file_input, + globals, + globals); if (result == nullptr) throw py::error_already_set(); Py_DECREF(result); @@ -127,15 +131,18 @@ public: int alive() { gc(); int total = 0; - for (const auto &p : _instances) - if (p.second > 0) + for (const auto &p : _instances) { + if (p.second > 0) { total += p.second; + } + } return total; } void value() {} // Recursion terminator // Takes one or more values, converts them to strings, then stores them. - template <typename T, typename... Tmore> void value(const T &v, Tmore &&...args) { + template <typename T, typename... Tmore> + void value(const T &v, Tmore &&...args) { std::ostringstream oss; oss << v; _values.push_back(oss.str()); @@ -145,19 +152,22 @@ public: // Move out stored values py::list values() { py::list l; - for (const auto &v : _values) l.append(py::cast(v)); + for (const auto &v : _values) { + l.append(py::cast(v)); + } _values.clear(); return l; } // Gets constructor stats from a C++ type index - static ConstructorStats& get(std::type_index type) { + static ConstructorStats &get(std::type_index type) { static std::unordered_map<std::type_index, ConstructorStats> all_cstats; return all_cstats[type]; } // Gets constructor stats from a C++ type - template <typename T> static ConstructorStats& get() { + template <typename T> + static ConstructorStats &get() { #if defined(PYPY_VERSION) gc(); #endif @@ -165,11 +175,12 @@ public: } // Gets constructor stats from a Python class - static ConstructorStats& get(py::object class_) { + static ConstructorStats &get(py::object class_) { auto &internals = py::detail::get_internals(); const std::type_index *t1 = nullptr, *t2 = nullptr; try { - auto *type_info = internals.registered_types_py.at((PyTypeObject *) class_.ptr()).at(0); + auto *type_info + = internals.registered_types_py.at((PyTypeObject *) class_.ptr()).at(0); for (auto &p : internals.registered_types_cpp) { if (p.second == type_info) { if (t1) { @@ -179,17 +190,23 @@ public: t1 = &p.first; } } + } catch (const std::out_of_range &) { + } + if (!t1) { + throw std::runtime_error("Unknown class passed to ConstructorStats::get()"); } - catch (const std::out_of_range&) {} - if (!t1) throw std::runtime_error("Unknown class passed to ConstructorStats::get()"); auto &cs1 = get(*t1); - // If we have both a t1 and t2 match, one is probably the trampoline class; return whichever - // has more constructions (typically one or the other will be 0) + // If we have both a t1 and t2 match, one is probably the trampoline class; return + // whichever has more constructions (typically one or the other will be 0) if (t2) { auto &cs2 = get(*t2); - int cs1_total = cs1.default_constructions + cs1.copy_constructions + cs1.move_constructions + (int) cs1._values.size(); - int cs2_total = cs2.default_constructions + cs2.copy_constructions + cs2.move_constructions + (int) cs2._values.size(); - if (cs2_total > cs1_total) return cs2; + int cs1_total = cs1.default_constructions + cs1.copy_constructions + + cs1.move_constructions + (int) cs1._values.size(); + int cs2_total = cs2.default_constructions + cs2.copy_constructions + + cs2.move_constructions + (int) cs2._values.size(); + if (cs2_total > cs1_total) { + return cs2; + } } return cs1; } @@ -198,78 +215,108 @@ public: // To track construction/destruction, you need to call these methods from the various // constructors/operators. The ones that take extra values record the given values in the // constructor stats values for later inspection. -template <class T> void track_copy_created(T *inst) { ConstructorStats::get<T>().copy_created(inst); } -template <class T> void track_move_created(T *inst) { ConstructorStats::get<T>().move_created(inst); } -template <class T, typename... Values> void track_copy_assigned(T *, Values &&...values) { +template <class T> +void track_copy_created(T *inst) { + ConstructorStats::get<T>().copy_created(inst); +} +template <class T> +void track_move_created(T *inst) { + ConstructorStats::get<T>().move_created(inst); +} +template <class T, typename... Values> +void track_copy_assigned(T *, Values &&...values) { auto &cst = ConstructorStats::get<T>(); cst.copy_assignments++; cst.value(std::forward<Values>(values)...); } -template <class T, typename... Values> void track_move_assigned(T *, Values &&...values) { +template <class T, typename... Values> +void track_move_assigned(T *, Values &&...values) { auto &cst = ConstructorStats::get<T>(); cst.move_assignments++; cst.value(std::forward<Values>(values)...); } -template <class T, typename... Values> void track_default_created(T *inst, Values &&...values) { +template <class T, typename... Values> +void track_default_created(T *inst, Values &&...values) { auto &cst = ConstructorStats::get<T>(); cst.default_created(inst); cst.value(std::forward<Values>(values)...); } -template <class T, typename... Values> void track_created(T *inst, Values &&...values) { +template <class T, typename... Values> +void track_created(T *inst, Values &&...values) { auto &cst = ConstructorStats::get<T>(); cst.created(inst); cst.value(std::forward<Values>(values)...); } -template <class T, typename... Values> void track_destroyed(T *inst) { +template <class T, typename... Values> +void track_destroyed(T *inst) { ConstructorStats::get<T>().destroyed(inst); } -template <class T, typename... Values> void track_values(T *, Values &&...values) { +template <class T, typename... Values> +void track_values(T *, Values &&...values) { ConstructorStats::get<T>().value(std::forward<Values>(values)...); } /// Don't cast pointers to Python, print them as strings inline const char *format_ptrs(const char *p) { return p; } template <typename T> -py::str format_ptrs(T *p) { return "{:#x}"_s.format(reinterpret_cast<std::uintptr_t>(p)); } +py::str format_ptrs(T *p) { + return "{:#x}"_s.format(reinterpret_cast<std::uintptr_t>(p)); +} template <typename T> -auto format_ptrs(T &&x) -> decltype(std::forward<T>(x)) { return std::forward<T>(x); } +auto format_ptrs(T &&x) -> decltype(std::forward<T>(x)) { + return std::forward<T>(x); +} template <class T, typename... Output> void print_constr_details(T *inst, const std::string &action, Output &&...output) { - py::print("###", py::type_id<T>(), "@", format_ptrs(inst), action, + py::print("###", + py::type_id<T>(), + "@", + format_ptrs(inst), + action, format_ptrs(std::forward<Output>(output))...); } // Verbose versions of the above: -template <class T, typename... Values> void print_copy_created(T *inst, Values &&...values) { // NB: this prints, but doesn't store, given values +template <class T, typename... Values> +void print_copy_created(T *inst, + Values &&...values) { // NB: this prints, but doesn't store, given values print_constr_details(inst, "created via copy constructor", values...); track_copy_created(inst); } -template <class T, typename... Values> void print_move_created(T *inst, Values &&...values) { // NB: this prints, but doesn't store, given values +template <class T, typename... Values> +void print_move_created(T *inst, + Values &&...values) { // NB: this prints, but doesn't store, given values print_constr_details(inst, "created via move constructor", values...); track_move_created(inst); } -template <class T, typename... Values> void print_copy_assigned(T *inst, Values &&...values) { +template <class T, typename... Values> +void print_copy_assigned(T *inst, Values &&...values) { print_constr_details(inst, "assigned via copy assignment", values...); track_copy_assigned(inst, values...); } -template <class T, typename... Values> void print_move_assigned(T *inst, Values &&...values) { +template <class T, typename... Values> +void print_move_assigned(T *inst, Values &&...values) { print_constr_details(inst, "assigned via move assignment", values...); track_move_assigned(inst, values...); } -template <class T, typename... Values> void print_default_created(T *inst, Values &&...values) { +template <class T, typename... Values> +void print_default_created(T *inst, Values &&...values) { print_constr_details(inst, "created via default constructor", values...); track_default_created(inst, values...); } -template <class T, typename... Values> void print_created(T *inst, Values &&...values) { +template <class T, typename... Values> +void print_created(T *inst, Values &&...values) { print_constr_details(inst, "created", values...); track_created(inst, values...); } -template <class T, typename... Values> void print_destroyed(T *inst, Values &&...values) { // Prints but doesn't store given values +template <class T, typename... Values> +void print_destroyed(T *inst, Values &&...values) { // Prints but doesn't store given values print_constr_details(inst, "destroyed", values...); track_destroyed(inst); } -template <class T, typename... Values> void print_values(T *inst, Values &&...values) { +template <class T, typename... Values> +void print_values(T *inst, Values &&...values) { print_constr_details(inst, ":", values...); track_values(inst, values...); } diff --git a/tests/cross_module_gil_utils.cpp b/tests/cross_module_gil_utils.cpp index 07db9f6e..7c20849d 100644 --- a/tests/cross_module_gil_utils.cpp +++ b/tests/cross_module_gil_utils.cpp @@ -6,8 +6,15 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance. #include <pybind11/pybind11.h> + #include <cstdint> +#include <string> +#include <thread> // This file mimics a DSO that makes pybind11 calls but does not define a // PYBIND11_MODULE. The purpose is to test that such a DSO can create a @@ -20,54 +27,82 @@ namespace { namespace py = pybind11; + void gil_acquire() { py::gil_scoped_acquire gil; } -constexpr char kModuleName[] = "cross_module_gil_utils"; +std::string gil_multi_acquire_release(unsigned bits) { + if ((bits & 0x1u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x2u) != 0u) { + py::gil_scoped_release gil; + } + if ((bits & 0x4u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x8u) != 0u) { + py::gil_scoped_release gil; + } + return PYBIND11_INTERNALS_ID; +} -#if PY_MAJOR_VERSION >= 3 -struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - kModuleName, - NULL, - 0, - NULL, - NULL, - NULL, - NULL, - NULL +struct CustomAutoGIL { + CustomAutoGIL() : gstate(PyGILState_Ensure()) {} + ~CustomAutoGIL() { PyGILState_Release(gstate); } + + PyGILState_STATE gstate; }; -#else -PyMethodDef module_methods[] = { - {NULL, NULL, 0, NULL} +struct CustomAutoNoGIL { + CustomAutoNoGIL() : save(PyEval_SaveThread()) {} + ~CustomAutoNoGIL() { PyEval_RestoreThread(save); } + + PyThreadState *save; }; -#endif -} // namespace +template <typename Acquire, typename Release> +void gil_acquire_inner() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; +} -extern "C" PYBIND11_EXPORT -#if PY_MAJOR_VERSION >= 3 -PyObject* PyInit_cross_module_gil_utils() -#else -void initcross_module_gil_utils() -#endif -{ +template <typename Acquire, typename Release> +void gil_acquire_nested() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; + auto thread = std::thread(&gil_acquire_inner<Acquire, Release>); + thread.join(); +} - PyObject* m = -#if PY_MAJOR_VERSION >= 3 - PyModule_Create(&moduledef); -#else - Py_InitModule(kModuleName, module_methods); -#endif +constexpr char kModuleName[] = "cross_module_gil_utils"; + +struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr}; + +} // namespace + +#define ADD_FUNCTION(Name, ...) \ + PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast<void *>(&__VA_ARGS__))); - if (m != NULL) { - static_assert( - sizeof(&gil_acquire) == sizeof(void*), - "Function pointer must have the same size as void*"); - PyModule_AddObject(m, "gil_acquire_funcaddr", - PyLong_FromVoidPtr(reinterpret_cast<void*>(&gil_acquire))); +extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { + + PyObject *m = PyModule_Create(&moduledef); + + if (m != nullptr) { + static_assert(sizeof(&gil_acquire) == sizeof(void *), + "Function pointer must have the same size as void*"); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner<CustomAutoGIL, CustomAutoNoGIL>) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested<CustomAutoGIL, CustomAutoNoGIL>) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner<py::gil_scoped_acquire, py::gil_scoped_release>) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested<py::gil_scoped_acquire, py::gil_scoped_release>) } -#if PY_MAJOR_VERSION >= 3 return m; -#endif } diff --git a/tests/cross_module_interleaved_error_already_set.cpp b/tests/cross_module_interleaved_error_already_set.cpp new file mode 100644 index 00000000..fdd9939e --- /dev/null +++ b/tests/cross_module_interleaved_error_already_set.cpp @@ -0,0 +1,51 @@ +/* + Copyright (c) 2022 Google LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include <pybind11/pybind11.h> + +// This file mimics a DSO that makes pybind11 calls but does not define a PYBIND11_MODULE, +// so that the first call of cross_module_error_already_set() triggers the first call of +// pybind11::detail::get_internals(). + +namespace { + +namespace py = pybind11; + +void interleaved_error_already_set() { + PyErr_SetString(PyExc_RuntimeError, "1st error."); + try { + throw py::error_already_set(); + } catch (const py::error_already_set &) { + // The 2nd error could be conditional in a real application. + PyErr_SetString(PyExc_RuntimeError, "2nd error."); + } // Here the 1st error is destroyed before the 2nd error is fetched. + // The error_already_set dtor triggers a pybind11::detail::get_internals() + // call via pybind11::gil_scoped_acquire. + if (PyErr_Occurred()) { + throw py::error_already_set(); + } +} + +constexpr char kModuleName[] = "cross_module_interleaved_error_already_set"; + +struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr}; + +} // namespace + +extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_interleaved_error_already_set() { + PyObject *m = PyModule_Create(&moduledef); + if (m != nullptr) { + static_assert(sizeof(&interleaved_error_already_set) == sizeof(void *), + "Function pointer must have the same size as void *"); + PyModule_AddObject( + m, + "funcaddr", + PyLong_FromVoidPtr(reinterpret_cast<void *>(&interleaved_error_already_set))); + } + return m; +} diff --git a/tests/eigen_tensor_avoid_stl_array.cpp b/tests/eigen_tensor_avoid_stl_array.cpp new file mode 100644 index 00000000..eacc9e9b --- /dev/null +++ b/tests/eigen_tensor_avoid_stl_array.cpp @@ -0,0 +1,14 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#ifndef EIGEN_AVOID_STL_ARRAY +# define EIGEN_AVOID_STL_ARRAY +#endif + +#include "test_eigen_tensor.inl" + +PYBIND11_MODULE(eigen_tensor_avoid_stl_array, m) { eigen_tensor_test::test_module(m); } diff --git a/tests/env.py b/tests/env.py index 5cded441..7eea5a3b 100644 --- a/tests/env.py +++ b/tests/env.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- import platform import sys +import pytest + LINUX = sys.platform.startswith("linux") MACOS = sys.platform.startswith("darwin") WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") @@ -9,6 +10,18 @@ WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") CPYTHON = platform.python_implementation() == "CPython" PYPY = platform.python_implementation() == "PyPy" -PY2 = sys.version_info.major == 2 -PY = sys.version_info +def deprecated_call(): + """ + pytest.deprecated_call() seems broken in pytest<3.9.x; concretely, it + doesn't work on CPython 3.8.0 with pytest==3.3.2 on Ubuntu 18.04 (#2922). + + This is a narrowed reimplementation of the following PR :( + https://github.com/pytest-dev/pytest/pull/4104 + """ + # TODO: Remove this when testing requires pytest>=3.9. + pieces = pytest.__version__.split(".") + pytest_major_minor = (int(pieces[0]), int(pieces[1])) + if pytest_major_minor < (3, 9): + return pytest.warns((DeprecationWarning, PendingDeprecationWarning)) + return pytest.deprecated_call() diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index cbd4bff1..57387dd8 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import contextlib import os import string @@ -13,6 +12,16 @@ import zipfile DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) +PKGCONFIG = """\ +prefix=${{pcfiledir}}/../../ +includedir=${{prefix}}/include + +Name: pybind11 +Description: Seamless operability between C++11 and Python +Version: {VERSION} +Cflags: -I${{includedir}} +""" + main_headers = { "include/pybind11/attr.h", @@ -25,6 +34,7 @@ main_headers = { "include/pybind11/embed.h", "include/pybind11/eval.h", "include/pybind11/functional.h", + "include/pybind11/gil.h", "include/pybind11/iostream.h", "include/pybind11/numpy.h", "include/pybind11/operators.h", @@ -33,6 +43,7 @@ main_headers = { "include/pybind11/pytypes.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/type_caster_pyobject_ptr.h", } detail_headers = { @@ -41,9 +52,20 @@ detail_headers = { "include/pybind11/detail/descr.h", "include/pybind11/detail/init.h", "include/pybind11/detail/internals.h", + "include/pybind11/detail/type_caster_base.h", "include/pybind11/detail/typeid.h", } +eigen_headers = { + "include/pybind11/eigen/common.h", + "include/pybind11/eigen/matrix.h", + "include/pybind11/eigen/tensor.h", +} + +stl_headers = { + "include/pybind11/stl/filesystem.h", +} + cmake_files = { "share/cmake/pybind11/FindPythonLibsNew.cmake", "share/cmake/pybind11/pybind11Common.cmake", @@ -54,19 +76,21 @@ cmake_files = { "share/cmake/pybind11/pybind11Tools.cmake", } +pkgconfig_files = { + "share/pkgconfig/pybind11.pc", +} + py_files = { "__init__.py", "__main__.py", "_version.py", - "_version.pyi", "commands.py", "py.typed", "setup_helpers.py", - "setup_helpers.pyi", } -headers = main_headers | detail_headers -src_files = headers | cmake_files +headers = main_headers | detail_headers | eigen_headers | stl_headers +src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files @@ -75,9 +99,12 @@ sdist_files = { "pybind11/include", "pybind11/include/pybind11", "pybind11/include/pybind11/detail", + "pybind11/include/pybind11/eigen", + "pybind11/include/pybind11/stl", "pybind11/share", "pybind11/share/cmake", "pybind11/share/cmake/pybind11", + "pybind11/share/pkgconfig", "pyproject.toml", "setup.cfg", "setup.py", @@ -85,6 +112,7 @@ sdist_files = { "MANIFEST.in", "README.rst", "PKG-INFO", + "SECURITY.md", } local_sdist_files = { @@ -97,105 +125,104 @@ local_sdist_files = { } -def test_build_sdist(monkeypatch, tmpdir): +def read_tz_file(tar: tarfile.TarFile, name: str) -> bytes: + start = tar.getnames()[0] + "/" + inner_file = tar.extractfile(tar.getmember(f"{start}{name}")) + assert inner_file + with contextlib.closing(inner_file) as f: + return f.read() + +def normalize_line_endings(value: bytes) -> bytes: + return value.replace(os.linesep.encode("utf-8"), b"\n") + + +def test_build_sdist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - out = subprocess.check_output( - [ - sys.executable, - "setup.py", - "sdist", - "--formats=tar", - "--dist-dir", - str(tmpdir), - ] + subprocess.run( + [sys.executable, "-m", "build", "--sdist", f"--outdir={tmpdir}"], check=True ) - if hasattr(out, "decode"): - out = out.decode() - (sdist,) = tmpdir.visit("*.tar") + (sdist,) = tmpdir.visit("*.tar.gz") - with tarfile.open(str(sdist)) as tar: + with tarfile.open(str(sdist), "r:gz") as tar: start = tar.getnames()[0] + "/" version = start[9:-1] - simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]} - with contextlib.closing( - tar.extractfile(tar.getmember(start + "setup.py")) - ) as f: - setup_py = f.read() + setup_py = read_tz_file(tar, "setup.py") + pyproject_toml = read_tz_file(tar, "pyproject.toml") + pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc") + cmake_cfg = read_tz_file( + tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake" + ) - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pyproject.toml")) - ) as f: - pyproject_toml = f.read() + assert ( + 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' + in cmake_cfg.decode("utf-8") + ) - files = set("pybind11/{}".format(n) for n in all_files) + files = {f"pybind11/{n}" for n in all_files} files |= sdist_files - files |= set("pybind11{}".format(n) for n in local_sdist_files) + files |= {f"pybind11{n}" for n in local_sdist_files} files.add("pybind11.egg-info/entry_points.txt") files.add("pybind11.egg-info/requires.txt") assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: contents = ( - string.Template(f.read().decode()) + string.Template(f.read().decode("utf-8")) .substitute(version=version, extra_cmd="") - .encode() + .encode("utf-8") ) - assert setup_py == contents + assert setup_py == contents with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f: contents = f.read() - assert pyproject_toml == contents + assert pyproject_toml == contents + simple_version = ".".join(version.split(".")[:3]) + pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8") + assert normalize_line_endings(pkgconfig) == pkgconfig_expected -def test_build_global_dist(monkeypatch, tmpdir): +def test_build_global_dist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") - - out = subprocess.check_output( - [ - sys.executable, - "setup.py", - "sdist", - "--formats=tar", - "--dist-dir", - str(tmpdir), - ] + subprocess.run( + [sys.executable, "-m", "build", "--sdist", "--outdir", str(tmpdir)], check=True ) - if hasattr(out, "decode"): - out = out.decode() - (sdist,) = tmpdir.visit("*.tar") + (sdist,) = tmpdir.visit("*.tar.gz") - with tarfile.open(str(sdist)) as tar: + with tarfile.open(str(sdist), "r:gz") as tar: start = tar.getnames()[0] + "/" version = start[16:-1] - simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:]) + simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]} - with contextlib.closing( - tar.extractfile(tar.getmember(start + "setup.py")) - ) as f: - setup_py = f.read() + setup_py = read_tz_file(tar, "setup.py") + pyproject_toml = read_tz_file(tar, "pyproject.toml") + pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc") + cmake_cfg = read_tz_file( + tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake" + ) - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pyproject.toml")) - ) as f: - pyproject_toml = f.read() + assert ( + 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' + in cmake_cfg.decode("utf-8") + ) - files = set("pybind11/{}".format(n) for n in all_files) + files = {f"pybind11/{n}" for n in all_files} files |= sdist_files - files |= set("pybind11_global{}".format(n) for n in local_sdist_files) + files |= {f"pybind11_global{n}" for n in local_sdist_files} assert simpler == files with open(os.path.join(MAIN_DIR, "tools", "setup_global.py.in"), "rb") as f: contents = ( string.Template(f.read().decode()) .substitute(version=version, extra_cmd="") - .encode() + .encode("utf-8") ) assert setup_py == contents @@ -203,17 +230,21 @@ def test_build_global_dist(monkeypatch, tmpdir): contents = f.read() assert pyproject_toml == contents + simple_version = ".".join(version.split(".")[:3]) + pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8") + assert normalize_line_endings(pkgconfig) == pkgconfig_expected + def tests_build_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - subprocess.check_output( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True ) (wheel,) = tmpdir.visit("*.whl") - files = set("pybind11/{}".format(n) for n in all_files) + files = {f"pybind11/{n}" for n in all_files} files |= { "dist-info/LICENSE", "dist-info/METADATA", @@ -226,10 +257,8 @@ def tests_build_wheel(monkeypatch, tmpdir): with zipfile.ZipFile(str(wheel)) as z: names = z.namelist() - trimmed = set(n for n in names if "dist-info" not in n) - trimmed |= set( - "dist-info/{}".format(n.split("/", 1)[-1]) for n in names if "dist-info" in n - ) + trimmed = {n for n in names if "dist-info" not in n} + trimmed |= {f"dist-info/{n.split('/', 1)[-1]}" for n in names if "dist-info" in n} assert files == trimmed @@ -237,14 +266,14 @@ def tests_build_global_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") - subprocess.check_output( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True ) (wheel,) = tmpdir.visit("*.whl") - files = set("data/data/{}".format(n) for n in src_files) - files |= set("data/headers/{}".format(n[8:]) for n in headers) + files = {f"data/data/{n}" for n in src_files} + files |= {f"data/headers/{n[8:]}" for n in headers} files |= { "dist-info/LICENSE", "dist-info/METADATA", @@ -257,6 +286,6 @@ def tests_build_global_wheel(monkeypatch, tmpdir): names = z.namelist() beginning = names[0].split("/", 1)[0].rsplit(".", 1)[0] - trimmed = set(n[len(beginning) + 1 :] for n in names) + trimmed = {n[len(beginning) + 1 :] for n in names} assert files == trimmed diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 0d8bd0e4..d5d3093b 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- import os -import sys import subprocess +import sys from textwrap import dedent import pytest DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) +WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin") @pytest.mark.parametrize("parallel", [False, True]) @@ -18,7 +18,7 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): (tmpdir / "setup.py").write_text( dedent( - u"""\ + f"""\ import sys sys.path.append({MAIN_DIR!r}) @@ -51,13 +51,13 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): ext_modules=ext_modules, ) """ - ).format(MAIN_DIR=MAIN_DIR, std=std, parallel=parallel), + ), encoding="ascii", ) (tmpdir / "main.cpp").write_text( dedent( - u"""\ + """\ #include <pybind11/pybind11.h> int f(int x) { @@ -71,13 +71,20 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): encoding="ascii", ) - subprocess.check_call( + out = subprocess.check_output( [sys.executable, "setup.py", "build_ext", "--inplace"], - stdout=sys.stdout, - stderr=sys.stderr, ) + if not WIN: + assert b"-g0" in out + out = subprocess.check_output( + [sys.executable, "setup.py", "build_ext", "--inplace", "--force"], + env=dict(os.environ, CFLAGS="-g"), + ) + if not WIN: + assert b"-g0" not in out # Debug helper printout, normally hidden + print(out) for item in tmpdir.listdir(): print(item.basename) @@ -88,7 +95,7 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): (tmpdir / "test.py").write_text( dedent( - u"""\ + """\ import simple_setup assert simple_setup.f(3) == 9 """ @@ -99,3 +106,46 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): subprocess.check_call( [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr ) + + +def test_intree_extensions(monkeypatch, tmpdir): + monkeypatch.syspath_prepend(MAIN_DIR) + + from pybind11.setup_helpers import intree_extensions + + monkeypatch.chdir(tmpdir) + root = tmpdir + root.ensure_dir() + subdir = root / "dir" + subdir.ensure_dir() + src = subdir / "ext.cpp" + src.ensure() + relpath = src.relto(tmpdir) + (ext,) = intree_extensions([relpath]) + assert ext.name == "ext" + subdir.ensure("__init__.py") + (ext,) = intree_extensions([relpath]) + assert ext.name == "dir.ext" + + +def test_intree_extensions_package_dir(monkeypatch, tmpdir): + monkeypatch.syspath_prepend(MAIN_DIR) + + from pybind11.setup_helpers import intree_extensions + + monkeypatch.chdir(tmpdir) + root = tmpdir / "src" + root.ensure_dir() + subdir = root / "dir" + subdir.ensure_dir() + src = subdir / "ext.cpp" + src.ensure() + (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"": "src"}) + assert ext.name == "dir.ext" + (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"foo": "src"}) + assert ext.name == "foo.dir.ext" + subdir.ensure("__init__.py") + (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"": "src"}) + assert ext.name == "dir.ext" + (ext,) = intree_extensions([src.relto(tmpdir)], package_dir={"foo": "src"}) + assert ext.name == "foo.dir.ext" diff --git a/tests/local_bindings.h b/tests/local_bindings.h index 22537b13..01d27853 100644 --- a/tests/local_bindings.h +++ b/tests/local_bindings.h @@ -1,10 +1,13 @@ #pragma once #include "pybind11_tests.h" +#include <utility> + /// Simple class used to test py::local: -template <int> class LocalBase { +template <int> +class LocalBase { public: - LocalBase(int i) : i(i) { } + explicit LocalBase(int i) : i(i) {} int i = -1; }; @@ -33,32 +36,57 @@ using NonLocalVec2 = std::vector<NonLocal2>; using NonLocalMap = std::unordered_map<std::string, NonLocalType>; using NonLocalMap2 = std::unordered_map<std::string, uint8_t>; +// Exception that will be caught via the module local translator. +class LocalException : public std::exception { +public: + explicit LocalException(const char *m) : message{m} {} + const char *what() const noexcept override { return message.c_str(); } + +private: + std::string message = ""; +}; + +// Exception that will be registered with register_local_exception_translator +class LocalSimpleException : public std::exception { +public: + explicit LocalSimpleException(const char *m) : message{m} {} + const char *what() const noexcept override { return message.c_str(); } + +private: + std::string message = ""; +}; + PYBIND11_MAKE_OPAQUE(LocalVec); PYBIND11_MAKE_OPAQUE(LocalVec2); PYBIND11_MAKE_OPAQUE(LocalMap); PYBIND11_MAKE_OPAQUE(NonLocalVec); -//PYBIND11_MAKE_OPAQUE(NonLocalVec2); // same type as LocalVec2 +// PYBIND11_MAKE_OPAQUE(NonLocalVec2); // same type as LocalVec2 PYBIND11_MAKE_OPAQUE(NonLocalMap); PYBIND11_MAKE_OPAQUE(NonLocalMap2); - // Simple bindings (used with the above): template <typename T, int Adjust = 0, typename... Args> -py::class_<T> bind_local(Args && ...args) { - return py::class_<T>(std::forward<Args>(args)...) - .def(py::init<int>()) - .def("get", [](T &i) { return i.i + Adjust; }); +py::class_<T> bind_local(Args &&...args) { + return py::class_<T>(std::forward<Args>(args)...).def(py::init<int>()).def("get", [](T &i) { + return i.i + Adjust; + }); }; // Simulate a foreign library base class (to match the example in the docs): namespace pets { class Pet { public: - Pet(std::string name) : name_(name) {} + explicit Pet(std::string name) : name_(std::move(name)) {} std::string name_; - const std::string &name() { return name_; } + const std::string &name() const { return name_; } }; } // namespace pets -struct MixGL { int i; MixGL(int i) : i{i} {} }; -struct MixGL2 { int i; MixGL2(int i) : i{i} {} }; +struct MixGL { + int i; + explicit MixGL(int i) : i{i} {} +}; +struct MixGL2 { + int i; + explicit MixGL2(int i) : i{i} {} +}; diff --git a/tests/object.h b/tests/object.h index 9235f19c..564dd4a7 100644 --- a/tests/object.h +++ b/tests/object.h @@ -1,8 +1,9 @@ #if !defined(__OBJECT_H) -#define __OBJECT_H +# define __OBJECT_H -#include <atomic> -#include "constructor_stats.h" +# include "constructor_stats.h" + +# include <atomic> /// Reference counted object base class class Object { @@ -27,20 +28,23 @@ public: */ void decRef(bool dealloc = true) const { --m_refCount; - if (m_refCount == 0 && dealloc) + if (m_refCount == 0 && dealloc) { delete this; - else if (m_refCount < 0) + } else if (m_refCount < 0) { throw std::runtime_error("Internal error: reference count < 0!"); + } } virtual std::string toString() const = 0; + protected: /** \brief Virtual protected deconstructor. * (Will only be called by \ref ref) */ virtual ~Object() { print_destroyed(this); } + private: - mutable std::atomic<int> m_refCount { 0 }; + mutable std::atomic<int> m_refCount{0}; }; // Tag class used to track constructions of ref objects. When we track constructors, below, we @@ -59,80 +63,105 @@ class ref_tag {}; * * \ingroup libcore */ -template <typename T> class ref { +template <typename T> +class ref { public: /// Create a nullptr reference - ref() : m_ptr(nullptr) { print_default_created(this); track_default_created((ref_tag*) this); } + ref() : m_ptr(nullptr) { + print_default_created(this); + track_default_created((ref_tag *) this); + } /// Construct a reference from a pointer - ref(T *ptr) : m_ptr(ptr) { - if (m_ptr) ((Object *) m_ptr)->incRef(); - - print_created(this, "from pointer", m_ptr); track_created((ref_tag*) this, "from pointer"); + explicit ref(T *ptr) : m_ptr(ptr) { + if (m_ptr) { + ((Object *) m_ptr)->incRef(); + } + print_created(this, "from pointer", m_ptr); + track_created((ref_tag *) this, "from pointer"); } /// Copy constructor ref(const ref &r) : m_ptr(r.m_ptr) { - if (m_ptr) + if (m_ptr) { ((Object *) m_ptr)->incRef(); + } - print_copy_created(this, "with pointer", m_ptr); track_copy_created((ref_tag*) this); + print_copy_created(this, "with pointer", m_ptr); + track_copy_created((ref_tag *) this); } /// Move constructor - ref(ref &&r) : m_ptr(r.m_ptr) { + ref(ref &&r) noexcept : m_ptr(r.m_ptr) { r.m_ptr = nullptr; - print_move_created(this, "with pointer", m_ptr); track_move_created((ref_tag*) this); + print_move_created(this, "with pointer", m_ptr); + track_move_created((ref_tag *) this); } /// Destroy this reference ~ref() { - if (m_ptr) + if (m_ptr) { ((Object *) m_ptr)->decRef(); + } - print_destroyed(this); track_destroyed((ref_tag*) this); + print_destroyed(this); + track_destroyed((ref_tag *) this); } /// Move another reference into the current one - ref& operator=(ref&& r) { - print_move_assigned(this, "pointer", r.m_ptr); track_move_assigned((ref_tag*) this); + ref &operator=(ref &&r) noexcept { + print_move_assigned(this, "pointer", r.m_ptr); + track_move_assigned((ref_tag *) this); - if (*this == r) + if (*this == r) { return *this; - if (m_ptr) + } + if (m_ptr) { ((Object *) m_ptr)->decRef(); + } m_ptr = r.m_ptr; r.m_ptr = nullptr; return *this; } /// Overwrite this reference with another reference - ref& operator=(const ref& r) { - print_copy_assigned(this, "pointer", r.m_ptr); track_copy_assigned((ref_tag*) this); + ref &operator=(const ref &r) { + if (this == &r) { + return *this; + } + print_copy_assigned(this, "pointer", r.m_ptr); + track_copy_assigned((ref_tag *) this); - if (m_ptr == r.m_ptr) + if (m_ptr == r.m_ptr) { return *this; - if (m_ptr) + } + if (m_ptr) { ((Object *) m_ptr)->decRef(); + } m_ptr = r.m_ptr; - if (m_ptr) + if (m_ptr) { ((Object *) m_ptr)->incRef(); + } return *this; } /// Overwrite this reference with a pointer to another object - ref& operator=(T *ptr) { - print_values(this, "assigned pointer"); track_values((ref_tag*) this, "assigned pointer"); + ref &operator=(T *ptr) { + print_values(this, "assigned pointer"); + track_values((ref_tag *) this, "assigned pointer"); - if (m_ptr == ptr) + if (m_ptr == ptr) { return *this; - if (m_ptr) + } + if (m_ptr) { ((Object *) m_ptr)->decRef(); + } m_ptr = ptr; - if (m_ptr) + if (m_ptr) { ((Object *) m_ptr)->incRef(); + } return *this; } @@ -143,31 +172,32 @@ public: bool operator!=(const ref &r) const { return m_ptr != r.m_ptr; } /// Compare this reference with a pointer - bool operator==(const T* ptr) const { return m_ptr == ptr; } + bool operator==(const T *ptr) const { return m_ptr == ptr; } /// Compare this reference with a pointer - bool operator!=(const T* ptr) const { return m_ptr != ptr; } + bool operator!=(const T *ptr) const { return m_ptr != ptr; } /// Access the object referenced by this reference - T* operator->() { return m_ptr; } + T *operator->() { return m_ptr; } /// Access the object referenced by this reference - const T* operator->() const { return m_ptr; } + const T *operator->() const { return m_ptr; } /// Return a C++ reference to the referenced object - T& operator*() { return *m_ptr; } + T &operator*() { return *m_ptr; } /// Return a const C++ reference to the referenced object - const T& operator*() const { return *m_ptr; } + const T &operator*() const { return *m_ptr; } /// Return a pointer to the referenced object - operator T* () { return m_ptr; } + explicit operator T *() { return m_ptr; } /// Return a const pointer to the referenced object - T* get_ptr() { return m_ptr; } + T *get_ptr() { return m_ptr; } /// Return a pointer to the referenced object - const T* get_ptr() const { return m_ptr; } + const T *get_ptr() const { return m_ptr; } + private: T *m_ptr; }; diff --git a/tests/pybind11_cross_module_tests.cpp b/tests/pybind11_cross_module_tests.cpp index f705e310..9379f3f2 100644 --- a/tests/pybind11_cross_module_tests.cpp +++ b/tests/pybind11_cross_module_tests.cpp @@ -7,10 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "local_bindings.h" #include <pybind11/stl_bind.h> + +#include "local_bindings.h" +#include "pybind11_tests.h" +#include "test_exceptions.h" + #include <numeric> +#include <utility> PYBIND11_MODULE(pybind11_cross_module_tests, m) { m.doc() = "pybind11 cross-module test module"; @@ -25,17 +29,46 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { bind_local<ExternalType2>(m, "ExternalType2", py::module_local()); // test_exceptions.py - m.def("raise_runtime_error", []() { PyErr_SetString(PyExc_RuntimeError, "My runtime error"); throw py::error_already_set(); }); - m.def("raise_value_error", []() { PyErr_SetString(PyExc_ValueError, "My value error"); throw py::error_already_set(); }); + py::register_local_exception<LocalSimpleException>(m, "LocalSimpleException"); + m.def("raise_runtime_error", []() { + PyErr_SetString(PyExc_RuntimeError, "My runtime error"); + throw py::error_already_set(); + }); + m.def("raise_value_error", []() { + PyErr_SetString(PyExc_ValueError, "My value error"); + throw py::error_already_set(); + }); m.def("throw_pybind_value_error", []() { throw py::value_error("pybind11 value error"); }); m.def("throw_pybind_type_error", []() { throw py::type_error("pybind11 type error"); }); m.def("throw_stop_iteration", []() { throw py::stop_iteration(); }); + m.def("throw_local_error", []() { throw LocalException("just local"); }); + m.def("throw_local_simple_error", []() { throw LocalSimpleException("external mod"); }); + py::register_exception_translator([](std::exception_ptr p) { + try { + if (p) { + std::rethrow_exception(p); + } + } catch (const shared_exception &e) { + PyErr_SetString(PyExc_KeyError, e.what()); + } + }); + + // translate the local exception into a key error but only in this module + py::register_local_exception_translator([](std::exception_ptr p) { + try { + if (p) { + std::rethrow_exception(p); + } + } catch (const LocalException &e) { + PyErr_SetString(PyExc_KeyError, e.what()); + } + }); // test_local_bindings.py // Local to both: - bind_local<LocalType, 1>(m, "LocalType", py::module_local()) - .def("get2", [](LocalType &t) { return t.i + 2; }) - ; + bind_local<LocalType, 1>(m, "LocalType", py::module_local()).def("get2", [](LocalType &t) { + return t.i + 2; + }); // Can only be called with our python type: m.def("local_value", [](LocalType &l) { return l.i; }); @@ -43,9 +76,7 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { // test_nonlocal_failure // This registration will fail (global registration when LocalFail is already registered // globally in the main test module): - m.def("register_nonlocal", [m]() { - bind_local<NonLocalType, 0>(m, "NonLocalType"); - }); + m.def("register_nonlocal", [m]() { bind_local<NonLocalType, 0>(m, "NonLocalType"); }); // test_stl_bind_local // stl_bind.h binders defaults to py::module_local if the types are local or converting: @@ -55,27 +86,21 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { // test_stl_bind_global // and global if the type (or one of the types, for the map) is global (so these will fail, // assuming pybind11_tests is already loaded): - m.def("register_nonlocal_vec", [m]() { - py::bind_vector<NonLocalVec>(m, "NonLocalVec"); - }); - m.def("register_nonlocal_map", [m]() { - py::bind_map<NonLocalMap>(m, "NonLocalMap"); - }); + m.def("register_nonlocal_vec", [m]() { py::bind_vector<NonLocalVec>(m, "NonLocalVec"); }); + m.def("register_nonlocal_map", [m]() { py::bind_map<NonLocalMap>(m, "NonLocalMap"); }); // The default can, however, be overridden to global using `py::module_local()` or // `py::module_local(false)`. // Explicitly made local: py::bind_vector<NonLocalVec2>(m, "NonLocalVec2", py::module_local()); // Explicitly made global (and so will fail to bind): - m.def("register_nonlocal_map2", [m]() { - py::bind_map<NonLocalMap2>(m, "NonLocalMap2", py::module_local(false)); - }); + m.def("register_nonlocal_map2", + [m]() { py::bind_map<NonLocalMap2>(m, "NonLocalMap2", py::module_local(false)); }); // test_mixed_local_global // We try this both with the global type registered first and vice versa (the order shouldn't // matter). - m.def("register_mixed_global_local", [m]() { - bind_local<MixedGlobalLocal, 200>(m, "MixedGlobalLocal", py::module_local()); - }); + m.def("register_mixed_global_local", + [m]() { bind_local<MixedGlobalLocal, 200>(m, "MixedGlobalLocal", py::module_local()); }); m.def("register_mixed_local_global", [m]() { bind_local<MixedLocalGlobal, 2000>(m, "MixedLocalGlobal", py::module_local(false)); }); @@ -83,25 +108,26 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { m.def("get_mixed_lg", [](int i) { return MixedLocalGlobal(i); }); // test_internal_locals_differ - m.def("local_cpp_types_addr", []() { return (uintptr_t) &py::detail::registered_local_types_cpp(); }); + m.def("local_cpp_types_addr", + []() { return (uintptr_t) &py::detail::get_local_internals().registered_types_cpp; }); // test_stl_caster_vs_stl_bind py::bind_vector<std::vector<int>>(m, "VectorInt"); - m.def("load_vector_via_binding", [](std::vector<int> &v) { - return std::accumulate(v.begin(), v.end(), 0); - }); + m.def("load_vector_via_binding", + [](std::vector<int> &v) { return std::accumulate(v.begin(), v.end(), 0); }); // test_cross_module_calls m.def("return_self", [](LocalVec *v) { return v; }); m.def("return_copy", [](const LocalVec &v) { return LocalVec(v); }); - class Dog : public pets::Pet { public: Dog(std::string name) : Pet(name) {}; }; - py::class_<pets::Pet>(m, "Pet", py::module_local()) - .def("name", &pets::Pet::name); + class Dog : public pets::Pet { + public: + explicit Dog(std::string name) : Pet(std::move(name)) {} + }; + py::class_<pets::Pet>(m, "Pet", py::module_local()).def("name", &pets::Pet::name); // Binding for local extending class: - py::class_<Dog, pets::Pet>(m, "Dog") - .def(py::init<std::string>()); + py::class_<Dog, pets::Pet>(m, "Dog").def(py::init<std::string>()); m.def("pet_name", [](pets::Pet &p) { return p.name(); }); py::class_<MixGL>(m, "MixGL", py::module_local()).def(py::init<int>()); @@ -118,6 +144,6 @@ PYBIND11_MODULE(pybind11_cross_module_tests, m) { // test_missing_header_message // The main module already includes stl.h, but we need to test the error message // which appears when this header is missing. - m.def("missing_header_arg", [](std::vector<float>) { }); + m.def("missing_header_arg", [](const std::vector<float> &) {}); m.def("missing_header_return", []() { return std::vector<float>(); }); } diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 439cd401..62403464 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -8,6 +8,7 @@ */ #include "pybind11_tests.h" + #include "constructor_stats.h" #include <functional> @@ -31,9 +32,7 @@ std::list<std::function<void(py::module_ &)>> &initializers() { return inits; } -test_initializer::test_initializer(Initializer init) { - initializers().emplace_back(init); -} +test_initializer::test_initializer(Initializer init) { initializers().emplace_back(init); } test_initializer::test_initializer(const char *submodule_name, Initializer init) { initializers().emplace_back([=](py::module_ &parent) { @@ -51,26 +50,58 @@ void bind_ConstructorStats(py::module_ &m) { .def_readwrite("move_assignments", &ConstructorStats::move_assignments) .def_readwrite("copy_constructions", &ConstructorStats::copy_constructions) .def_readwrite("move_constructions", &ConstructorStats::move_constructions) - .def_static("get", (ConstructorStats &(*)(py::object)) &ConstructorStats::get, py::return_value_policy::reference_internal) + .def_static("get", + (ConstructorStats & (*) (py::object)) & ConstructorStats::get, + py::return_value_policy::reference_internal) - // Not exactly ConstructorStats, but related: expose the internal pybind number of registered instances - // to allow instance cleanup checks (invokes a GC first) + // Not exactly ConstructorStats, but related: expose the internal pybind number of + // registered instances to allow instance cleanup checks (invokes a GC first) .def_static("detail_reg_inst", []() { ConstructorStats::gc(); return py::detail::get_internals().registered_instances.size(); - }) - ; + }); +} + +const char *cpp_std() { + return +#if defined(PYBIND11_CPP20) + "C++20"; +#elif defined(PYBIND11_CPP17) + "C++17"; +#elif defined(PYBIND11_CPP14) + "C++14"; +#else + "C++11"; +#endif } PYBIND11_MODULE(pybind11_tests, m) { m.doc() = "pybind11 test module"; + // Intentionally kept minimal to not create a maintenance chore + // ("just enough" to be conclusive). +#if defined(_MSC_FULL_VER) + m.attr("compiler_info") = "MSVC " PYBIND11_TOSTRING(_MSC_FULL_VER); +#elif defined(__VERSION__) + m.attr("compiler_info") = __VERSION__; +#else + m.attr("compiler_info") = py::none(); +#endif + m.attr("cpp_std") = cpp_std(); + m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = +#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + true; +#else + false; +#endif + bind_ConstructorStats(m); -#if !defined(NDEBUG) - m.attr("debug_enabled") = true; +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + m.attr("detailed_error_messages_enabled") = true; #else - m.attr("debug_enabled") = false; + m.attr("detailed_error_messages_enabled") = false; #endif py::class_<UserType>(m, "UserType", "A `py::class_` type for testing") @@ -79,13 +110,14 @@ PYBIND11_MODULE(pybind11_tests, m) { .def("get_value", &UserType::value, "Get value using a method") .def("set_value", &UserType::set, "Set value using a method") .def_property("value", &UserType::value, &UserType::set, "Get/set value using a property") - .def("__repr__", [](const UserType& u) { return "UserType({})"_s.format(u.value()); }); + .def("__repr__", [](const UserType &u) { return "UserType({})"_s.format(u.value()); }); py::class_<IncType, UserType>(m, "IncType") .def(py::init<>()) .def(py::init<int>()) - .def("__repr__", [](const IncType& u) { return "IncType({})"_s.format(u.value()); }); + .def("__repr__", [](const IncType &u) { return "IncType({})"_s.format(u.value()); }); - for (const auto &initializer : initializers()) + for (const auto &initializer : initializers()) { initializer(m); + } } diff --git a/tests/pybind11_tests.h b/tests/pybind11_tests.h index ccb05297..a7c00c2f 100644 --- a/tests/pybind11_tests.h +++ b/tests/pybind11_tests.h @@ -1,11 +1,7 @@ #pragma once -#include <pybind11/pybind11.h> -#include <pybind11/eval.h> -#if defined(_MSC_VER) && _MSC_VER < 1910 -// We get some really long type names here which causes MSVC 2015 to emit warnings -# pragma warning(disable: 4503) // warning C4503: decorated name length exceeded, name was truncated -#endif +#include <pybind11/eval.h> +#include <pybind11/pybind11.h> namespace py = pybind11; using namespace pybind11::literals; @@ -14,24 +10,23 @@ class test_initializer { using Initializer = void (*)(py::module_ &); public: - test_initializer(Initializer init); + explicit test_initializer(Initializer init); test_initializer(const char *submodule_name, Initializer init); }; -#define TEST_SUBMODULE(name, variable) \ - void test_submodule_##name(py::module_ &); \ - test_initializer name(#name, test_submodule_##name); \ - void test_submodule_##name(py::module_ &variable) - +#define TEST_SUBMODULE(name, variable) \ + void test_submodule_##name(py::module_ &); \ + test_initializer name(#name, test_submodule_##name); \ + void test_submodule_##name(py::module_ &(variable)) /// Dummy type which is not exported anywhere -- something to trigger a conversion error -struct UnregisteredType { }; +struct UnregisteredType {}; /// A user-defined type which is exported and can be used by any test class UserType { public: UserType() = default; - UserType(int i) : i(i) { } + explicit UserType(int i) : i(i) {} int value() const { return i; } void set(int set) { i = set; } @@ -45,7 +40,7 @@ class IncType : public UserType { public: using UserType::UserType; IncType() = default; - IncType(const IncType &other) : IncType(other.value() + 1) { } + IncType(const IncType &other) : IncType(other.value() + 1) {} IncType(IncType &&) = delete; IncType &operator=(const IncType &) = delete; IncType &operator=(IncType &&) = delete; @@ -57,16 +52,21 @@ union IntFloat { float f; }; -/// Custom cast-only type that casts to a string "rvalue" or "lvalue" depending on the cast context. -/// Used to test recursive casters (e.g. std::tuple, stl containers). +/// Custom cast-only type that casts to a string "rvalue" or "lvalue" depending on the cast +/// context. Used to test recursive casters (e.g. std::tuple, stl containers). struct RValueCaster {}; PYBIND11_NAMESPACE_BEGIN(pybind11) PYBIND11_NAMESPACE_BEGIN(detail) -template<> class type_caster<RValueCaster> { +template <> +class type_caster<RValueCaster> { public: - PYBIND11_TYPE_CASTER(RValueCaster, _("RValueCaster")); - static handle cast(RValueCaster &&, return_value_policy, handle) { return py::str("rvalue").release(); } - static handle cast(const RValueCaster &, return_value_policy, handle) { return py::str("lvalue").release(); } + PYBIND11_TYPE_CASTER(RValueCaster, const_name("RValueCaster")); + static handle cast(RValueCaster &&, return_value_policy, handle) { + return py::str("rvalue").release(); + } + static handle cast(const RValueCaster &, return_value_policy, handle) { + return py::str("lvalue").release(); + } }; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) @@ -80,5 +80,6 @@ void ignoreOldStyleInitWarnings(F &&body) { with warnings.catch_warnings(): warnings.filterwarnings("ignore", message=message, category=FutureWarning) body() - )", py::dict(py::arg("body") = py::cpp_function(body))); + )", + py::dict(py::arg("body") = py::cpp_function(body))); } diff --git a/tests/pytest.ini b/tests/pytest.ini index c47cbe9c..792ba361 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,17 +1,20 @@ [pytest] -minversion = 3.1 +minversion = 3.10 norecursedirs = test_* extra_* xfail_strict = True addopts = - # show summary of skipped tests - -rs + # show summary of tests + -ra # capture only Python print and C++ py::print, but not C output (low-level Python errors) --capture=sys - # enable all warnings - -Wa + # Show local info when a failure occurs + --showlocals +log_cli_level = info filterwarnings = # make warnings into errors but ignore certain third-party extension issues error + # somehow, some DeprecationWarnings do not get turned into errors + always::DeprecationWarning # importing scipy submodules on some version of Python ignore::ImportWarning # bogus numpy ABI warning (see numpy/#432) diff --git a/tests/requirements.txt b/tests/requirements.txt index 8768fc1f..4ba10111 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,10 +1,9 @@ ---extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/ -numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" -numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6" -numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10" -pytest==4.6.9; python_version<"3.5" -pytest==6.1.2; python_version=="3.5" -pytest==6.2.1; python_version>="3.6" +build==0.8.0 +numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7" +numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6" +numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10" +numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" and python_version<"3.11" +pytest==7.0.0 pytest-timeout -scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6" -scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10" +scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" +scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" diff --git a/tests/test_async.cpp b/tests/test_async.cpp index e6e01d72..a5d72246 100644 --- a/tests/test_async.cpp +++ b/tests/test_async.cpp @@ -11,12 +11,11 @@ TEST_SUBMODULE(async_module, m) { struct DoesNotSupportAsync {}; - py::class_<DoesNotSupportAsync>(m, "DoesNotSupportAsync") - .def(py::init<>()); + py::class_<DoesNotSupportAsync>(m, "DoesNotSupportAsync").def(py::init<>()); struct SupportsAsync {}; py::class_<SupportsAsync>(m, "SupportsAsync") .def(py::init<>()) - .def("__await__", [](const SupportsAsync& self) -> py::object { + .def("__await__", [](const SupportsAsync &self) -> py::object { static_cast<void>(self); py::object loop = py::module_::import("asyncio.events").attr("get_event_loop")(); py::object f = loop.attr("create_future")(); diff --git a/tests/test_async.py b/tests/test_async.py index df4489c4..83a4c503 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- import pytest asyncio = pytest.importorskip("asyncio") m = pytest.importorskip("pybind11_tests.async_module") -@pytest.fixture +@pytest.fixture() def event_loop(): loop = asyncio.new_event_loop() yield loop @@ -17,7 +16,7 @@ async def get_await_result(x): def test_await(event_loop): - assert 5 == event_loop.run_until_complete(get_await_result(m.SupportsAsync())) + assert event_loop.run_until_complete(get_await_result(m.SupportsAsync())) == 5 def test_await_missing(event_loop): diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 46eabf39..b5b8c355 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -7,27 +7,66 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" +#include <pybind11/complex.h> #include <pybind11/stl.h> +#include "constructor_stats.h" +#include "pybind11_tests.h" + TEST_SUBMODULE(buffers, m) { + m.attr("long_double_and_double_have_same_size") = (sizeof(long double) == sizeof(double)); + + m.def("format_descriptor_format_buffer_info_equiv", + [](const std::string &cpp_name, const py::buffer &buffer) { + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *format_table = new std::map<std::string, std::string>; + static auto *equiv_table + = new std::map<std::string, bool (py::buffer_info::*)() const>; + if (format_table->empty()) { +#define PYBIND11_ASSIGN_HELPER(...) \ + (*format_table)[#__VA_ARGS__] = py::format_descriptor<__VA_ARGS__>::format(); \ + (*equiv_table)[#__VA_ARGS__] = &py::buffer_info::item_type_is_equivalent_to<__VA_ARGS__>; + PYBIND11_ASSIGN_HELPER(PyObject *) + PYBIND11_ASSIGN_HELPER(bool) + PYBIND11_ASSIGN_HELPER(std::int8_t) + PYBIND11_ASSIGN_HELPER(std::uint8_t) + PYBIND11_ASSIGN_HELPER(std::int16_t) + PYBIND11_ASSIGN_HELPER(std::uint16_t) + PYBIND11_ASSIGN_HELPER(std::int32_t) + PYBIND11_ASSIGN_HELPER(std::uint32_t) + PYBIND11_ASSIGN_HELPER(std::int64_t) + PYBIND11_ASSIGN_HELPER(std::uint64_t) + PYBIND11_ASSIGN_HELPER(float) + PYBIND11_ASSIGN_HELPER(double) + PYBIND11_ASSIGN_HELPER(long double) + PYBIND11_ASSIGN_HELPER(std::complex<float>) + PYBIND11_ASSIGN_HELPER(std::complex<double>) + PYBIND11_ASSIGN_HELPER(std::complex<long double>) +#undef PYBIND11_ASSIGN_HELPER + } + return std::pair<std::string, bool>( + (*format_table)[cpp_name], (buffer.request().*((*equiv_table)[cpp_name]))()); + }); + // test_from_python / test_to_python: class Matrix { public: Matrix(py::ssize_t rows, py::ssize_t cols) : m_rows(rows), m_cols(cols) { print_created(this, std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); - m_data = new float[(size_t) (rows*cols)]; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + m_data = new float[(size_t) (rows * cols)]; memset(m_data, 0, sizeof(float) * (size_t) (rows * cols)); } Matrix(const Matrix &s) : m_rows(s.m_rows), m_cols(s.m_cols) { - print_copy_created(this, std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); + print_copy_created(this, + std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) m_data = new float[(size_t) (m_rows * m_cols)]; memcpy(m_data, s.m_data, sizeof(float) * (size_t) (m_rows * m_cols)); } - Matrix(Matrix &&s) : m_rows(s.m_rows), m_cols(s.m_cols), m_data(s.m_data) { + Matrix(Matrix &&s) noexcept : m_rows(s.m_rows), m_cols(s.m_cols), m_data(s.m_data) { print_move_created(this); s.m_rows = 0; s.m_cols = 0; @@ -35,12 +74,17 @@ TEST_SUBMODULE(buffers, m) { } ~Matrix() { - print_destroyed(this, std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); + print_destroyed(this, + std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); delete[] m_data; } Matrix &operator=(const Matrix &s) { - print_copy_assigned(this, std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); + if (this == &s) { + return *this; + } + print_copy_assigned(this, + std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); delete[] m_data; m_rows = s.m_rows; m_cols = s.m_cols; @@ -49,28 +93,34 @@ TEST_SUBMODULE(buffers, m) { return *this; } - Matrix &operator=(Matrix &&s) { - print_move_assigned(this, std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); + Matrix &operator=(Matrix &&s) noexcept { + print_move_assigned(this, + std::to_string(m_rows) + "x" + std::to_string(m_cols) + " matrix"); if (&s != this) { delete[] m_data; - m_rows = s.m_rows; m_cols = s.m_cols; m_data = s.m_data; - s.m_rows = 0; s.m_cols = 0; s.m_data = nullptr; + m_rows = s.m_rows; + m_cols = s.m_cols; + m_data = s.m_data; + s.m_rows = 0; + s.m_cols = 0; + s.m_data = nullptr; } return *this; } float operator()(py::ssize_t i, py::ssize_t j) const { - return m_data[(size_t) (i*m_cols + j)]; + return m_data[(size_t) (i * m_cols + j)]; } float &operator()(py::ssize_t i, py::ssize_t j) { - return m_data[(size_t) (i*m_cols + j)]; + return m_data[(size_t) (i * m_cols + j)]; } float *data() { return m_data; } py::ssize_t rows() const { return m_rows; } py::ssize_t cols() const { return m_cols; } + private: py::ssize_t m_rows; py::ssize_t m_cols; @@ -79,51 +129,51 @@ TEST_SUBMODULE(buffers, m) { py::class_<Matrix>(m, "Matrix", py::buffer_protocol()) .def(py::init<py::ssize_t, py::ssize_t>()) /// Construct from a buffer - .def(py::init([](py::buffer const b) { + .def(py::init([](const py::buffer &b) { py::buffer_info info = b.request(); - if (info.format != py::format_descriptor<float>::format() || info.ndim != 2) + if (info.format != py::format_descriptor<float>::format() || info.ndim != 2) { throw std::runtime_error("Incompatible buffer format!"); + } - auto v = new Matrix(info.shape[0], info.shape[1]); + auto *v = new Matrix(info.shape[0], info.shape[1]); memcpy(v->data(), info.ptr, sizeof(float) * (size_t) (v->rows() * v->cols())); return v; })) - .def("rows", &Matrix::rows) - .def("cols", &Matrix::cols) + .def("rows", &Matrix::rows) + .def("cols", &Matrix::cols) /// Bare bones interface - .def("__getitem__", [](const Matrix &m, std::pair<py::ssize_t, py::ssize_t> i) { - if (i.first >= m.rows() || i.second >= m.cols()) - throw py::index_error(); - return m(i.first, i.second); - }) - .def("__setitem__", [](Matrix &m, std::pair<py::ssize_t, py::ssize_t> i, float v) { - if (i.first >= m.rows() || i.second >= m.cols()) - throw py::index_error(); - m(i.first, i.second) = v; - }) - /// Provide buffer access - .def_buffer([](Matrix &m) -> py::buffer_info { + .def("__getitem__", + [](const Matrix &m, std::pair<py::ssize_t, py::ssize_t> i) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + return m(i.first, i.second); + }) + .def("__setitem__", + [](Matrix &m, std::pair<py::ssize_t, py::ssize_t> i, float v) { + if (i.first >= m.rows() || i.second >= m.cols()) { + throw py::index_error(); + } + m(i.first, i.second) = v; + }) + /// Provide buffer access + .def_buffer([](Matrix &m) -> py::buffer_info { return py::buffer_info( - m.data(), /* Pointer to buffer */ - { m.rows(), m.cols() }, /* Buffer dimensions */ - { sizeof(float) * size_t(m.cols()), /* Strides (in bytes) for each index */ - sizeof(float) } - ); - }) - ; - + m.data(), /* Pointer to buffer */ + {m.rows(), m.cols()}, /* Buffer dimensions */ + {sizeof(float) * size_t(m.cols()), /* Strides (in bytes) for each index */ + sizeof(float)}); + }); // test_inherited_protocol class SquareMatrix : public Matrix { public: - SquareMatrix(py::ssize_t n) : Matrix(n, n) { } + explicit SquareMatrix(py::ssize_t n) : Matrix(n, n) {} }; // Derived classes inherit the buffer protocol and the buffer access function - py::class_<SquareMatrix, Matrix>(m, "SquareMatrix") - .def(py::init<py::ssize_t>()); - + py::class_<SquareMatrix, Matrix>(m, "SquareMatrix").def(py::init<py::ssize_t>()); // test_pointer_to_member_fn // Tests that passing a pointer to member to the base class works in @@ -132,8 +182,8 @@ TEST_SUBMODULE(buffers, m) { int32_t value = 0; py::buffer_info get_buffer_info() { - return py::buffer_info(&value, sizeof(value), - py::format_descriptor<int32_t>::format(), 1); + return py::buffer_info( + &value, sizeof(value), py::format_descriptor<int32_t>::format(), 1); } }; py::class_<Buffer>(m, "Buffer", py::buffer_protocol()) @@ -141,7 +191,6 @@ TEST_SUBMODULE(buffers, m) { .def_readwrite("value", &Buffer::value) .def_buffer(&Buffer::get_buffer_info); - class ConstBuffer { std::unique_ptr<int32_t> value; @@ -150,18 +199,18 @@ TEST_SUBMODULE(buffers, m) { void set_value(int32_t v) { *value = v; } py::buffer_info get_buffer_info() const { - return py::buffer_info(value.get(), sizeof(*value), - py::format_descriptor<int32_t>::format(), 1); + return py::buffer_info( + value.get(), sizeof(*value), py::format_descriptor<int32_t>::format(), 1); } - ConstBuffer() : value(new int32_t{0}) { }; + ConstBuffer() : value(new int32_t{0}) {} }; py::class_<ConstBuffer>(m, "ConstBuffer", py::buffer_protocol()) .def(py::init<>()) .def_property("value", &ConstBuffer::get_value, &ConstBuffer::set_value) .def_buffer(&ConstBuffer::get_buffer_info); - struct DerivedBuffer : public Buffer { }; + struct DerivedBuffer : public Buffer {}; py::class_<DerivedBuffer>(m, "DerivedBuffer", py::buffer_protocol()) .def(py::init<>()) .def_readwrite("value", (int32_t DerivedBuffer::*) &DerivedBuffer::value) @@ -169,11 +218,9 @@ TEST_SUBMODULE(buffers, m) { struct BufferReadOnly { const uint8_t value = 0; - BufferReadOnly(uint8_t value): value(value) {} + explicit BufferReadOnly(uint8_t value) : value(value) {} - py::buffer_info get_buffer_info() { - return py::buffer_info(&value, 1); - } + py::buffer_info get_buffer_info() { return py::buffer_info(&value, 1); } }; py::class_<BufferReadOnly>(m, "BufferReadOnly", py::buffer_protocol()) .def(py::init<uint8_t>()) @@ -183,9 +230,7 @@ TEST_SUBMODULE(buffers, m) { uint8_t value = 0; bool readonly = false; - py::buffer_info get_buffer_info() { - return py::buffer_info(&value, 1, readonly); - } + py::buffer_info get_buffer_info() { return py::buffer_info(&value, 1, readonly); } }; py::class_<BufferReadOnlySelect>(m, "BufferReadOnlySelect", py::buffer_protocol()) .def(py::init<>()) @@ -204,11 +249,11 @@ TEST_SUBMODULE(buffers, m) { .def_readonly("strides", &py::buffer_info::strides) .def_readonly("readonly", &py::buffer_info::readonly) .def("__repr__", [](py::handle self) { - return py::str("itemsize={0.itemsize!r}, size={0.size!r}, format={0.format!r}, ndim={0.ndim!r}, shape={0.shape!r}, strides={0.strides!r}, readonly={0.readonly!r}").format(self); - }) - ; + return py::str("itemsize={0.itemsize!r}, size={0.size!r}, format={0.format!r}, " + "ndim={0.ndim!r}, shape={0.shape!r}, strides={0.strides!r}, " + "readonly={0.readonly!r}") + .format(self); + }); - m.def("get_buffer_info", [](py::buffer buffer) { - return buffer.request(); - }); + m.def("get_buffer_info", [](const py::buffer &buffer) { return buffer.request(); }); } diff --git a/tests/test_buffers.py b/tests/test_buffers.py index 50845758..63d9d869 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -1,17 +1,72 @@ -# -*- coding: utf-8 -*- +import ctypes import io import struct -import ctypes import pytest -import env # noqa: F401 - -from pybind11_tests import buffers as m +import env from pybind11_tests import ConstructorStats +from pybind11_tests import buffers as m np = pytest.importorskip("numpy") +if m.long_double_and_double_have_same_size: + # Determined by the compiler used to build the pybind11 tests + # (e.g. MSVC gets here, but MinGW might not). + np_float128 = None + np_complex256 = None +else: + # Determined by the compiler used to build numpy (e.g. MinGW). + np_float128 = getattr(np, *["float128"] * 2) + np_complex256 = getattr(np, *["complex256"] * 2) + +CPP_NAME_FORMAT_NP_DTYPE_TABLE = [ + ("PyObject *", "O", object), + ("bool", "?", np.bool_), + ("std::int8_t", "b", np.int8), + ("std::uint8_t", "B", np.uint8), + ("std::int16_t", "h", np.int16), + ("std::uint16_t", "H", np.uint16), + ("std::int32_t", "i", np.int32), + ("std::uint32_t", "I", np.uint32), + ("std::int64_t", "q", np.int64), + ("std::uint64_t", "Q", np.uint64), + ("float", "f", np.float32), + ("double", "d", np.float64), + ("long double", "g", np_float128), + ("std::complex<float>", "Zf", np.complex64), + ("std::complex<double>", "Zd", np.complex128), + ("std::complex<long double>", "Zg", np_complex256), +] +CPP_NAME_FORMAT_TABLE = [ + (cpp_name, format) + for cpp_name, format, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE + if np_dtype is not None +] +CPP_NAME_NP_DTYPE_TABLE = [ + (cpp_name, np_dtype) for cpp_name, _, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE +] + + +@pytest.mark.parametrize(("cpp_name", "np_dtype"), CPP_NAME_NP_DTYPE_TABLE) +def test_format_descriptor_format_buffer_info_equiv(cpp_name, np_dtype): + if np_dtype is None: + pytest.skip( + f"cpp_name=`{cpp_name}`: `long double` and `double` have same size." + ) + if isinstance(np_dtype, str): + pytest.skip(f"np.{np_dtype} does not exist.") + np_array = np.array([], dtype=np_dtype) + for other_cpp_name, expected_format in CPP_NAME_FORMAT_TABLE: + format, np_array_is_matching = m.format_descriptor_format_buffer_info_equiv( + other_cpp_name, np_array + ) + assert format == expected_format + if other_cpp_name == cpp_name: + assert np_array_is_matching + else: + assert not np_array_is_matching + def test_from_python(): with pytest.raises(RuntimeError) as excinfo: @@ -37,6 +92,10 @@ def test_from_python(): # https://foss.heptapod.net/pypy/pypy/-/issues/2444 +# TODO: fix on recent PyPy +@pytest.mark.xfail( + env.PYPY, reason="PyPy 7.3.7 doesn't clear this anymore", strict=False +) def test_to_python(): mat = m.Matrix(5, 4) assert memoryview(mat).shape == (5, 4) @@ -52,7 +111,8 @@ def test_to_python(): mat2 = np.array(mat, copy=False) assert mat2.shape == (5, 4) assert abs(mat2).sum() == 11 - assert mat2[2, 3] == 4 and mat2[3, 2] == 7 + assert mat2[2, 3] == 4 + assert mat2[3, 2] == 7 mat2[2, 3] = 5 assert mat2[2, 3] == 5 @@ -90,16 +150,16 @@ def test_pointer_to_member_fn(): def test_readonly_buffer(): buf = m.BufferReadOnly(0x64) view = memoryview(buf) - assert view[0] == b"d" if env.PY2 else 0x64 + assert view[0] == 0x64 assert view.readonly with pytest.raises(TypeError): - view[0] = b"\0" if env.PY2 else 0 + view[0] = 0 def test_selective_readonly_buffer(): buf = m.BufferReadOnlySelect() - memoryview(buf)[0] = b"d" if env.PY2 else 0x64 + memoryview(buf)[0] = 0x64 assert buf.value == 0x64 io.BytesIO(b"A").readinto(buf) @@ -107,7 +167,7 @@ def test_selective_readonly_buffer(): buf.readonly = True with pytest.raises(TypeError): - memoryview(buf)[0] = b"\0" if env.PY2 else 0 + memoryview(buf)[0] = 0 with pytest.raises(TypeError): io.BytesIO(b"1").readinto(buf) @@ -142,9 +202,6 @@ def test_ctypes_array_2d(): assert not info.readonly -@pytest.mark.skipif( - "env.PYPY and env.PY2", reason="PyPy2 bytes buffer not reported as readonly" -) def test_ctypes_from_buffer(): test_pystr = b"0123456789" for pyarray in (test_pystr, bytearray(test_pystr)): diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index f4e77567..0623b85d 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -7,83 +7,120 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/complex.h> -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4127) // warning C4127: Conditional expression is constant -#endif +#include "pybind11_tests.h" struct ConstRefCasted { - int tag; + int tag; }; PYBIND11_NAMESPACE_BEGIN(pybind11) PYBIND11_NAMESPACE_BEGIN(detail) template <> class type_caster<ConstRefCasted> { - public: - static constexpr auto name = _<ConstRefCasted>(); - - // Input is unimportant, a new value will always be constructed based on the - // cast operator. - bool load(handle, bool) { return true; } - - operator ConstRefCasted&&() { value = {1}; return std::move(value); } - operator ConstRefCasted&() { value = {2}; return value; } - operator ConstRefCasted*() { value = {3}; return &value; } - - operator const ConstRefCasted&() { value = {4}; return value; } - operator const ConstRefCasted*() { value = {5}; return &value; } - - // custom cast_op to explicitly propagate types to the conversion operators. - template <typename T_> - using cast_op_type = - /// const - conditional_t< - std::is_same<remove_reference_t<T_>, const ConstRefCasted*>::value, const ConstRefCasted*, - conditional_t< - std::is_same<T_, const ConstRefCasted&>::value, const ConstRefCasted&, - /// non-const - conditional_t< - std::is_same<remove_reference_t<T_>, ConstRefCasted*>::value, ConstRefCasted*, - conditional_t< - std::is_same<T_, ConstRefCasted&>::value, ConstRefCasted&, - /* else */ConstRefCasted&&>>>>; - - private: - ConstRefCasted value = {0}; +public: + static constexpr auto name = const_name<ConstRefCasted>(); + + // Input is unimportant, a new value will always be constructed based on the + // cast operator. + bool load(handle, bool) { return true; } + + explicit operator ConstRefCasted &&() { + value = {1}; + // NOLINTNEXTLINE(performance-move-const-arg) + return std::move(value); + } + explicit operator ConstRefCasted &() { + value = {2}; + return value; + } + explicit operator ConstRefCasted *() { + value = {3}; + return &value; + } + + explicit operator const ConstRefCasted &() { + value = {4}; + return value; + } + explicit operator const ConstRefCasted *() { + value = {5}; + return &value; + } + + // custom cast_op to explicitly propagate types to the conversion operators. + template <typename T_> + using cast_op_type = + /// const + conditional_t< + std::is_same<remove_reference_t<T_>, const ConstRefCasted *>::value, + const ConstRefCasted *, + conditional_t< + std::is_same<T_, const ConstRefCasted &>::value, + const ConstRefCasted &, + /// non-const + conditional_t<std::is_same<remove_reference_t<T_>, ConstRefCasted *>::value, + ConstRefCasted *, + conditional_t<std::is_same<T_, ConstRefCasted &>::value, + ConstRefCasted &, + /* else */ ConstRefCasted &&>>>>; + +private: + ConstRefCasted value = {0}; }; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) TEST_SUBMODULE(builtin_casters, m) { + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_MSVC(4127) + // test_simple_string m.def("string_roundtrip", [](const char *s) { return s; }); // test_unicode_conversion - // Some test characters in utf16 and utf32 encodings. The last one (the 𝐀) contains a null byte - char32_t a32 = 0x61 /*a*/, z32 = 0x7a /*z*/, ib32 = 0x203d /*‽*/, cake32 = 0x1f382 /*🎂*/, mathbfA32 = 0x1d400 /*𝐀*/; - char16_t b16 = 0x62 /*b*/, z16 = 0x7a, ib16 = 0x203d, cake16_1 = 0xd83c, cake16_2 = 0xdf82, mathbfA16_1 = 0xd835, mathbfA16_2 = 0xdc00; + // Some test characters in utf16 and utf32 encodings. The last one (the 𝐀) contains a null + // byte + char32_t a32 = 0x61 /*a*/, z32 = 0x7a /*z*/, ib32 = 0x203d /*‽*/, cake32 = 0x1f382 /*🎂*/, + mathbfA32 = 0x1d400 /*𝐀*/; + char16_t b16 = 0x62 /*b*/, z16 = 0x7a, ib16 = 0x203d, cake16_1 = 0xd83c, cake16_2 = 0xdf82, + mathbfA16_1 = 0xd835, mathbfA16_2 = 0xdc00; std::wstring wstr; - wstr.push_back(0x61); // a + wstr.push_back(0x61); // a wstr.push_back(0x2e18); // ⸘ - if (sizeof(wchar_t) == 2) { wstr.push_back(mathbfA16_1); wstr.push_back(mathbfA16_2); } // 𝐀, utf16 - else { wstr.push_back((wchar_t) mathbfA32); } // 𝐀, utf32 + if (sizeof(wchar_t) == 2) { + wstr.push_back(mathbfA16_1); + wstr.push_back(mathbfA16_2); + } // 𝐀, utf16 + else { + wstr.push_back((wchar_t) mathbfA32); + } // 𝐀, utf32 wstr.push_back(0x7a); // z - m.def("good_utf8_string", []() { return std::string((const char*)u8"Say utf8\u203d \U0001f382 \U0001d400"); }); // Say utf8‽ 🎂 𝐀 - m.def("good_utf16_string", [=]() { return std::u16string({ b16, ib16, cake16_1, cake16_2, mathbfA16_1, mathbfA16_2, z16 }); }); // b‽🎂𝐀z - m.def("good_utf32_string", [=]() { return std::u32string({ a32, mathbfA32, cake32, ib32, z32 }); }); // a𝐀🎂‽z + m.def("good_utf8_string", []() { + return std::string((const char *) u8"Say utf8\u203d \U0001f382 \U0001d400"); + }); // Say utf8‽ 🎂 𝐀 + m.def("good_utf16_string", [=]() { + return std::u16string({b16, ib16, cake16_1, cake16_2, mathbfA16_1, mathbfA16_2, z16}); + }); // b‽🎂𝐀z + m.def("good_utf32_string", [=]() { + return std::u32string({a32, mathbfA32, cake32, ib32, z32}); + }); // a𝐀🎂‽z m.def("good_wchar_string", [=]() { return wstr; }); // a‽𝐀z - m.def("bad_utf8_string", []() { return std::string("abc\xd0" "def"); }); - m.def("bad_utf16_string", [=]() { return std::u16string({ b16, char16_t(0xd800), z16 }); }); - // Under Python 2.7, invalid unicode UTF-32 characters don't appear to trigger UnicodeDecodeError - if (PY_MAJOR_VERSION >= 3) - m.def("bad_utf32_string", [=]() { return std::u32string({ a32, char32_t(0xd800), z32 }); }); - if (PY_MAJOR_VERSION >= 3 || sizeof(wchar_t) == 2) - m.def("bad_wchar_string", [=]() { return std::wstring({ wchar_t(0x61), wchar_t(0xd800) }); }); + m.def("bad_utf8_string", []() { + return std::string("abc\xd0" + "def"); + }); + m.def("bad_utf16_string", [=]() { return std::u16string({b16, char16_t(0xd800), z16}); }); + // Under Python 2.7, invalid unicode UTF-32 characters didn't appear to trigger + // UnicodeDecodeError + m.def("bad_utf32_string", [=]() { return std::u32string({a32, char32_t(0xd800), z32}); }); + if (sizeof(wchar_t) == 2) { + m.def("bad_wchar_string", [=]() { + return std::wstring({wchar_t(0x61), wchar_t(0xd800)}); + }); + } m.def("u8_Z", []() -> char { return 'Z'; }); m.def("u8_eacute", []() -> char { return '\xe9'; }); m.def("u16_ibang", [=]() -> char16_t { return ib16; }); @@ -101,12 +138,17 @@ TEST_SUBMODULE(builtin_casters, m) { // test_bytes_to_string m.def("strlen", [](char *s) { return strlen(s); }); - m.def("string_length", [](std::string s) { return s.length(); }); + m.def("string_length", [](const std::string &s) { return s.length(); }); #ifdef PYBIND11_HAS_U8STRING m.attr("has_u8string") = true; - m.def("good_utf8_u8string", []() { return std::u8string(u8"Say utf8\u203d \U0001f382 \U0001d400"); }); // Say utf8‽ 🎂 𝐀 - m.def("bad_utf8_u8string", []() { return std::u8string((const char8_t*)"abc\xd0" "def"); }); + m.def("good_utf8_u8string", []() { + return std::u8string(u8"Say utf8\u203d \U0001f382 \U0001d400"); + }); // Say utf8‽ 🎂 𝐀 + m.def("bad_utf8_u8string", []() { + return std::u8string((const char8_t *) "abc\xd0" + "def"); + }); m.def("u8_char8_Z", []() -> char8_t { return u8'Z'; }); @@ -118,21 +160,72 @@ TEST_SUBMODULE(builtin_casters, m) { // test_string_view #ifdef PYBIND11_HAS_STRING_VIEW m.attr("has_string_view") = true; - m.def("string_view_print", [](std::string_view s) { py::print(s, s.size()); }); + m.def("string_view_print", [](std::string_view s) { py::print(s, s.size()); }); m.def("string_view16_print", [](std::u16string_view s) { py::print(s, s.size()); }); m.def("string_view32_print", [](std::u32string_view s) { py::print(s, s.size()); }); - m.def("string_view_chars", [](std::string_view s) { py::list l; for (auto c : s) l.append((std::uint8_t) c); return l; }); - m.def("string_view16_chars", [](std::u16string_view s) { py::list l; for (auto c : s) l.append((int) c); return l; }); - m.def("string_view32_chars", [](std::u32string_view s) { py::list l; for (auto c : s) l.append((int) c); return l; }); - m.def("string_view_return", []() { return std::string_view((const char*)u8"utf8 secret \U0001f382"); }); - m.def("string_view16_return", []() { return std::u16string_view(u"utf16 secret \U0001f382"); }); - m.def("string_view32_return", []() { return std::u32string_view(U"utf32 secret \U0001f382"); }); - -# ifdef PYBIND11_HAS_U8STRING - m.def("string_view8_print", [](std::u8string_view s) { py::print(s, s.size()); }); - m.def("string_view8_chars", [](std::u8string_view s) { py::list l; for (auto c : s) l.append((std::uint8_t) c); return l; }); + m.def("string_view_chars", [](std::string_view s) { + py::list l; + for (auto c : s) { + l.append((std::uint8_t) c); + } + return l; + }); + m.def("string_view16_chars", [](std::u16string_view s) { + py::list l; + for (auto c : s) { + l.append((int) c); + } + return l; + }); + m.def("string_view32_chars", [](std::u32string_view s) { + py::list l; + for (auto c : s) { + l.append((int) c); + } + return l; + }); + m.def("string_view_return", + []() { return std::string_view((const char *) u8"utf8 secret \U0001f382"); }); + m.def("string_view16_return", + []() { return std::u16string_view(u"utf16 secret \U0001f382"); }); + m.def("string_view32_return", + []() { return std::u32string_view(U"utf32 secret \U0001f382"); }); + + // The inner lambdas here are to also test implicit conversion + using namespace std::literals; + m.def("string_view_bytes", + []() { return [](py::bytes b) { return b; }("abc \x80\x80 def"sv); }); + m.def("string_view_str", + []() { return [](py::str s) { return s; }("abc \342\200\275 def"sv); }); + m.def("string_view_from_bytes", + [](const py::bytes &b) { return [](std::string_view s) { return s; }(b); }); + m.def("string_view_memoryview", []() { + static constexpr auto val = "Have some \360\237\216\202"sv; + return py::memoryview::from_memory(val); + }); + +# ifdef PYBIND11_HAS_U8STRING + m.def("string_view8_print", [](std::u8string_view s) { py::print(s, s.size()); }); + m.def("string_view8_chars", [](std::u8string_view s) { + py::list l; + for (auto c : s) + l.append((std::uint8_t) c); + return l; + }); m.def("string_view8_return", []() { return std::u8string_view(u8"utf8 secret \U0001f382"); }); -# endif + m.def("string_view8_str", []() { return py::str{std::u8string_view{u8"abc ‽ def"}}; }); +# endif + + struct TypeWithBothOperatorStringAndStringView { + // NOLINTNEXTLINE(google-explicit-constructor) + operator std::string() const { return "success"; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator std::string_view() const { return "failure"; } + }; + m.def("bytes_from_type_with_both_operator_string_and_string_view", + []() { return py::bytes(TypeWithBothOperatorStringAndStringView()); }); + m.def("str_from_type_with_both_operator_string_and_string_view", + []() { return py::str(TypeWithBothOperatorStringAndStringView()); }); #endif // test_integer_casting @@ -143,45 +236,63 @@ TEST_SUBMODULE(builtin_casters, m) { // test_int_convert m.def("int_passthrough", [](int arg) { return arg; }); - m.def("int_passthrough_noconvert", [](int arg) { return arg; }, py::arg{}.noconvert()); + m.def( + "int_passthrough_noconvert", [](int arg) { return arg; }, py::arg{}.noconvert()); // test_tuple - m.def("pair_passthrough", [](std::pair<bool, std::string> input) { - return std::make_pair(input.second, input.first); - }, "Return a pair in reversed order"); - m.def("tuple_passthrough", [](std::tuple<bool, std::string, int> input) { - return std::make_tuple(std::get<2>(input), std::get<1>(input), std::get<0>(input)); - }, "Return a triple in reversed order"); + m.def( + "pair_passthrough", + [](const std::pair<bool, std::string> &input) { + return std::make_pair(input.second, input.first); + }, + "Return a pair in reversed order"); + m.def( + "tuple_passthrough", + [](std::tuple<bool, std::string, int> input) { + return std::make_tuple(std::get<2>(input), std::get<1>(input), std::get<0>(input)); + }, + "Return a triple in reversed order"); m.def("empty_tuple", []() { return std::tuple<>(); }); static std::pair<RValueCaster, RValueCaster> lvpair; static std::tuple<RValueCaster, RValueCaster, RValueCaster> lvtuple; - static std::pair<RValueCaster, std::tuple<RValueCaster, std::pair<RValueCaster, RValueCaster>>> lvnested; + static std::pair<RValueCaster, std::tuple<RValueCaster, std::pair<RValueCaster, RValueCaster>>> + lvnested; m.def("rvalue_pair", []() { return std::make_pair(RValueCaster{}, RValueCaster{}); }); m.def("lvalue_pair", []() -> const decltype(lvpair) & { return lvpair; }); - m.def("rvalue_tuple", []() { return std::make_tuple(RValueCaster{}, RValueCaster{}, RValueCaster{}); }); + m.def("rvalue_tuple", + []() { return std::make_tuple(RValueCaster{}, RValueCaster{}, RValueCaster{}); }); m.def("lvalue_tuple", []() -> const decltype(lvtuple) & { return lvtuple; }); m.def("rvalue_nested", []() { - return std::make_pair(RValueCaster{}, std::make_tuple(RValueCaster{}, std::make_pair(RValueCaster{}, RValueCaster{}))); }); + return std::make_pair( + RValueCaster{}, + std::make_tuple(RValueCaster{}, std::make_pair(RValueCaster{}, RValueCaster{}))); + }); m.def("lvalue_nested", []() -> const decltype(lvnested) & { return lvnested; }); - static std::pair<int, std::string> int_string_pair{2, "items"}; - m.def("int_string_pair", []() { return &int_string_pair; }); + m.def( + "int_string_pair", + []() { + // Using no-destructor idiom to side-step warnings from overzealous compilers. + static auto *int_string_pair = new std::pair<int, std::string>{2, "items"}; + return int_string_pair; + }, + py::return_value_policy::reference); // test_builtins_cast_return_none m.def("return_none_string", []() -> std::string * { return nullptr; }); - m.def("return_none_char", []() -> const char * { return nullptr; }); - m.def("return_none_bool", []() -> bool * { return nullptr; }); - m.def("return_none_int", []() -> int * { return nullptr; }); - m.def("return_none_float", []() -> float * { return nullptr; }); - m.def("return_none_pair", []() -> std::pair<int,int> * { return nullptr; }); + m.def("return_none_char", []() -> const char * { return nullptr; }); + m.def("return_none_bool", []() -> bool * { return nullptr; }); + m.def("return_none_int", []() -> int * { return nullptr; }); + m.def("return_none_float", []() -> float * { return nullptr; }); + m.def("return_none_pair", []() -> std::pair<int, int> * { return nullptr; }); // test_none_deferred m.def("defer_none_cstring", [](char *) { return false; }); - m.def("defer_none_cstring", [](py::none) { return true; }); + m.def("defer_none_cstring", [](const py::none &) { return true; }); m.def("defer_none_custom", [](UserType *) { return false; }); - m.def("defer_none_custom", [](py::none) { return true; }); + m.def("defer_none_custom", [](const py::none &) { return true; }); m.def("nodefer_none_void", [](void *) { return true; }); - m.def("nodefer_none_void", [](py::none) { return false; }); + m.def("nodefer_none_void", [](const py::none &) { return false; }); // test_void_caster m.def("load_nullptr_t", [](std::nullptr_t) {}); // not useful, but it should still compile @@ -191,7 +302,8 @@ TEST_SUBMODULE(builtin_casters, m) { // test_bool_caster m.def("bool_passthrough", [](bool arg) { return arg; }); - m.def("bool_passthrough_noconvert", [](bool arg) { return arg; }, py::arg{}.noconvert()); + m.def( + "bool_passthrough_noconvert", [](bool arg) { return arg; }, py::arg{}.noconvert()); // TODO: This should be disabled and fixed in future Intel compilers #if !defined(__INTEL_COMPILER) @@ -199,13 +311,15 @@ TEST_SUBMODULE(builtin_casters, m) { // When compiled with the Intel compiler, this results in segmentation faults when importing // the module. Tested with icc (ICC) 2021.1 Beta 20200827, this should be tested again when // a newer version of icc is available. - m.def("bool_passthrough_noconvert2", [](bool arg) { return arg; }, py::arg().noconvert()); + m.def( + "bool_passthrough_noconvert2", [](bool arg) { return arg; }, py::arg().noconvert()); #endif // test_reference_wrapper m.def("refwrap_builtin", [](std::reference_wrapper<int> p) { return 10 * p.get(); }); m.def("refwrap_usertype", [](std::reference_wrapper<UserType> p) { return p.get().value(); }); - m.def("refwrap_usertype_const", [](std::reference_wrapper<const UserType> p) { return p.get().value(); }); + m.def("refwrap_usertype_const", + [](std::reference_wrapper<const UserType> p) { return p.get().value(); }); m.def("refwrap_lvalue", []() -> std::reference_wrapper<UserType> { static UserType x(1); @@ -218,20 +332,23 @@ TEST_SUBMODULE(builtin_casters, m) { // Not currently supported (std::pair caster has return-by-value cast operator); // triggers static_assert failure. - //m.def("refwrap_pair", [](std::reference_wrapper<std::pair<int, int>>) { }); - - m.def("refwrap_list", [](bool copy) { - static IncType x1(1), x2(2); - py::list l; - for (auto &f : {std::ref(x1), std::ref(x2)}) { - l.append(py::cast(f, copy ? py::return_value_policy::copy - : py::return_value_policy::reference)); - } - return l; - }, "copy"_a); + // m.def("refwrap_pair", [](std::reference_wrapper<std::pair<int, int>>) { }); + + m.def( + "refwrap_list", + [](bool copy) { + static IncType x1(1), x2(2); + py::list l; + for (const auto &f : {std::ref(x1), std::ref(x2)}) { + l.append(py::cast( + f, copy ? py::return_value_policy::copy : py::return_value_policy::reference)); + } + return l; + }, + "copy"_a); m.def("refwrap_iiw", [](const IncType &w) { return w.value(); }); - m.def("refwrap_call_iiw", [](IncType &w, py::function f) { + m.def("refwrap_call_iiw", [](IncType &w, const py::function &f) { py::list l; l.append(f(std::ref(w))); l.append(f(std::cref(w))); @@ -245,12 +362,13 @@ TEST_SUBMODULE(builtin_casters, m) { // test_complex m.def("complex_cast", [](float x) { return "{}"_s.format(x); }); - m.def("complex_cast", [](std::complex<float> x) { return "({}, {})"_s.format(x.real(), x.imag()); }); + m.def("complex_cast", + [](std::complex<float> x) { return "({}, {})"_s.format(x.real(), x.imag()); }); // test int vs. long (Python 2) - m.def("int_cast", []() {return (int) 42;}); - m.def("long_cast", []() {return (long) 42;}); - m.def("longlong_cast", []() {return ULLONG_MAX;}); + m.def("int_cast", []() { return (int) 42; }); + m.def("long_cast", []() { return (long) 42; }); + m.def("longlong_cast", []() { return ULLONG_MAX; }); /// test void* cast operator m.def("test_void_caster", []() -> bool { @@ -261,11 +379,14 @@ TEST_SUBMODULE(builtin_casters, m) { // Tests const/non-const propagation in cast_op. m.def("takes", [](ConstRefCasted x) { return x.tag; }); - m.def("takes_move", [](ConstRefCasted&& x) { return x.tag; }); - m.def("takes_ptr", [](ConstRefCasted* x) { return x->tag; }); - m.def("takes_ref", [](ConstRefCasted& x) { return x.tag; }); + m.def("takes_move", [](ConstRefCasted &&x) { return x.tag; }); + m.def("takes_ptr", [](ConstRefCasted *x) { return x->tag; }); + m.def("takes_ref", [](ConstRefCasted &x) { return x.tag; }); m.def("takes_ref_wrap", [](std::reference_wrapper<ConstRefCasted> x) { return x.get().tag; }); - m.def("takes_const_ptr", [](const ConstRefCasted* x) { return x->tag; }); - m.def("takes_const_ref", [](const ConstRefCasted& x) { return x.tag; }); - m.def("takes_const_ref_wrap", [](std::reference_wrapper<const ConstRefCasted> x) { return x.get().tag; }); + m.def("takes_const_ptr", [](const ConstRefCasted *x) { return x->tag; }); + m.def("takes_const_ref", [](const ConstRefCasted &x) { return x.tag; }); + m.def("takes_const_ref_wrap", + [](std::reference_wrapper<const ConstRefCasted> x) { return x.get().tag; }); + + PYBIND11_WARNING_POP } diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index cb37dbcb..b1f57bdd 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- -import pytest +import sys -import env # noqa: F401 +import pytest +import env +from pybind11_tests import IncType, UserType from pybind11_tests import builtin_casters as m -from pybind11_tests import UserType, IncType def test_simple_string(): @@ -13,12 +13,12 @@ def test_simple_string(): def test_unicode_conversion(): """Tests unicode conversion and error reporting.""" - assert m.good_utf8_string() == u"Say utf8‽ 🎂 𝐀" - assert m.good_utf16_string() == u"b‽🎂𝐀z" - assert m.good_utf32_string() == u"a𝐀🎂‽z" - assert m.good_wchar_string() == u"a⸘𝐀z" + assert m.good_utf8_string() == "Say utf8‽ 🎂 𝐀" + assert m.good_utf16_string() == "b‽🎂𝐀z" + assert m.good_utf32_string() == "a𝐀🎂‽z" + assert m.good_wchar_string() == "a⸘𝐀z" if hasattr(m, "has_u8string"): - assert m.good_utf8_u8string() == u"Say utf8‽ 🎂 𝐀" + assert m.good_utf8_u8string() == "Say utf8‽ 🎂 𝐀" with pytest.raises(UnicodeDecodeError): m.bad_utf8_string() @@ -26,7 +26,7 @@ def test_unicode_conversion(): with pytest.raises(UnicodeDecodeError): m.bad_utf16_string() - # These are provided only if they actually fail (they don't when 32-bit and under Python 2.7) + # These are provided only if they actually fail (they don't when 32-bit) if hasattr(m, "bad_utf32_string"): with pytest.raises(UnicodeDecodeError): m.bad_utf32_string() @@ -38,10 +38,10 @@ def test_unicode_conversion(): m.bad_utf8_u8string() assert m.u8_Z() == "Z" - assert m.u8_eacute() == u"é" - assert m.u16_ibang() == u"‽" - assert m.u32_mathbfA() == u"𝐀" - assert m.wchar_heart() == u"♥" + assert m.u8_eacute() == "é" + assert m.u16_ibang() == "‽" + assert m.u32_mathbfA() == "𝐀" + assert m.wchar_heart() == "♥" if hasattr(m, "has_u8string"): assert m.u8_char8_Z() == "Z" @@ -50,72 +50,72 @@ def test_single_char_arguments(): """Tests failures for passing invalid inputs to char-accepting functions""" def toobig_message(r): - return "Character code point not in range({0:#x})".format(r) + return f"Character code point not in range({r:#x})" toolong_message = "Expected a character, but multi-character string found" - assert m.ord_char(u"a") == 0x61 # simple ASCII - assert m.ord_char_lv(u"b") == 0x62 + assert m.ord_char("a") == 0x61 # simple ASCII + assert m.ord_char_lv("b") == 0x62 assert ( - m.ord_char(u"é") == 0xE9 + m.ord_char("é") == 0xE9 ) # requires 2 bytes in utf-8, but can be stuffed in a char with pytest.raises(ValueError) as excinfo: - assert m.ord_char(u"Ā") == 0x100 # requires 2 bytes, doesn't fit in a char + assert m.ord_char("Ā") == 0x100 # requires 2 bytes, doesn't fit in a char assert str(excinfo.value) == toobig_message(0x100) with pytest.raises(ValueError) as excinfo: - assert m.ord_char(u"ab") + assert m.ord_char("ab") assert str(excinfo.value) == toolong_message - assert m.ord_char16(u"a") == 0x61 - assert m.ord_char16(u"é") == 0xE9 - assert m.ord_char16_lv(u"ê") == 0xEA - assert m.ord_char16(u"Ā") == 0x100 - assert m.ord_char16(u"‽") == 0x203D - assert m.ord_char16(u"♥") == 0x2665 - assert m.ord_char16_lv(u"♡") == 0x2661 + assert m.ord_char16("a") == 0x61 + assert m.ord_char16("é") == 0xE9 + assert m.ord_char16_lv("ê") == 0xEA + assert m.ord_char16("Ā") == 0x100 + assert m.ord_char16("‽") == 0x203D + assert m.ord_char16("♥") == 0x2665 + assert m.ord_char16_lv("♡") == 0x2661 with pytest.raises(ValueError) as excinfo: - assert m.ord_char16(u"🎂") == 0x1F382 # requires surrogate pair + assert m.ord_char16("🎂") == 0x1F382 # requires surrogate pair assert str(excinfo.value) == toobig_message(0x10000) with pytest.raises(ValueError) as excinfo: - assert m.ord_char16(u"aa") + assert m.ord_char16("aa") assert str(excinfo.value) == toolong_message - assert m.ord_char32(u"a") == 0x61 - assert m.ord_char32(u"é") == 0xE9 - assert m.ord_char32(u"Ā") == 0x100 - assert m.ord_char32(u"‽") == 0x203D - assert m.ord_char32(u"♥") == 0x2665 - assert m.ord_char32(u"🎂") == 0x1F382 + assert m.ord_char32("a") == 0x61 + assert m.ord_char32("é") == 0xE9 + assert m.ord_char32("Ā") == 0x100 + assert m.ord_char32("‽") == 0x203D + assert m.ord_char32("♥") == 0x2665 + assert m.ord_char32("🎂") == 0x1F382 with pytest.raises(ValueError) as excinfo: - assert m.ord_char32(u"aa") + assert m.ord_char32("aa") assert str(excinfo.value) == toolong_message - assert m.ord_wchar(u"a") == 0x61 - assert m.ord_wchar(u"é") == 0xE9 - assert m.ord_wchar(u"Ā") == 0x100 - assert m.ord_wchar(u"‽") == 0x203D - assert m.ord_wchar(u"♥") == 0x2665 + assert m.ord_wchar("a") == 0x61 + assert m.ord_wchar("é") == 0xE9 + assert m.ord_wchar("Ā") == 0x100 + assert m.ord_wchar("‽") == 0x203D + assert m.ord_wchar("♥") == 0x2665 if m.wchar_size == 2: with pytest.raises(ValueError) as excinfo: - assert m.ord_wchar(u"🎂") == 0x1F382 # requires surrogate pair + assert m.ord_wchar("🎂") == 0x1F382 # requires surrogate pair assert str(excinfo.value) == toobig_message(0x10000) else: - assert m.ord_wchar(u"🎂") == 0x1F382 + assert m.ord_wchar("🎂") == 0x1F382 with pytest.raises(ValueError) as excinfo: - assert m.ord_wchar(u"aa") + assert m.ord_wchar("aa") assert str(excinfo.value) == toolong_message if hasattr(m, "has_u8string"): - assert m.ord_char8(u"a") == 0x61 # simple ASCII - assert m.ord_char8_lv(u"b") == 0x62 + assert m.ord_char8("a") == 0x61 # simple ASCII + assert m.ord_char8_lv("b") == 0x62 assert ( - m.ord_char8(u"é") == 0xE9 + m.ord_char8("é") == 0xE9 ) # requires 2 bytes in utf-8, but can be stuffed in a char with pytest.raises(ValueError) as excinfo: - assert m.ord_char8(u"Ā") == 0x100 # requires 2 bytes, doesn't fit in a char + assert m.ord_char8("Ā") == 0x100 # requires 2 bytes, doesn't fit in a char assert str(excinfo.value) == toobig_message(0x100) with pytest.raises(ValueError) as excinfo: - assert m.ord_char8(u"ab") + assert m.ord_char8("ab") assert str(excinfo.value) == toolong_message @@ -124,18 +124,22 @@ def test_bytes_to_string(): one-way: the only way to return bytes to Python is via the pybind11::bytes class.""" # Issue #816 - def to_bytes(s): - b = s if env.PY2 else s.encode("utf8") - assert isinstance(b, bytes) - return b - - assert m.strlen(to_bytes("hi")) == 2 - assert m.string_length(to_bytes("world")) == 5 - assert m.string_length(to_bytes("a\x00b")) == 3 - assert m.strlen(to_bytes("a\x00b")) == 1 # C-string limitation + assert m.strlen(b"hi") == 2 + assert m.string_length(b"world") == 5 + assert m.string_length(b"a\x00b") == 3 + assert m.strlen(b"a\x00b") == 1 # C-string limitation # passing in a utf8 encoded string should work - assert m.string_length(u"💩".encode("utf8")) == 4 + assert m.string_length("💩".encode()) == 4 + + +def test_bytearray_to_string(): + """Tests the ability to pass bytearray to C++ string-accepting functions""" + assert m.string_length(bytearray(b"Hi")) == 2 + assert m.strlen(bytearray(b"bytearray")) == 9 + assert m.string_length(bytearray()) == 0 + assert m.string_length(bytearray("🦜", "utf-8", "strict")) == 4 + assert m.string_length(bytearray(b"\x80")) == 1 @pytest.mark.skipif(not hasattr(m, "has_string_view"), reason="no <string_view>") @@ -143,26 +147,26 @@ def test_string_view(capture): """Tests support for C++17 string_view arguments and return values""" assert m.string_view_chars("Hi") == [72, 105] assert m.string_view_chars("Hi 🎂") == [72, 105, 32, 0xF0, 0x9F, 0x8E, 0x82] - assert m.string_view16_chars(u"Hi 🎂") == [72, 105, 32, 0xD83C, 0xDF82] - assert m.string_view32_chars(u"Hi 🎂") == [72, 105, 32, 127874] + assert m.string_view16_chars("Hi 🎂") == [72, 105, 32, 0xD83C, 0xDF82] + assert m.string_view32_chars("Hi 🎂") == [72, 105, 32, 127874] if hasattr(m, "has_u8string"): assert m.string_view8_chars("Hi") == [72, 105] - assert m.string_view8_chars(u"Hi 🎂") == [72, 105, 32, 0xF0, 0x9F, 0x8E, 0x82] + assert m.string_view8_chars("Hi 🎂") == [72, 105, 32, 0xF0, 0x9F, 0x8E, 0x82] - assert m.string_view_return() == u"utf8 secret 🎂" - assert m.string_view16_return() == u"utf16 secret 🎂" - assert m.string_view32_return() == u"utf32 secret 🎂" + assert m.string_view_return() == "utf8 secret 🎂" + assert m.string_view16_return() == "utf16 secret 🎂" + assert m.string_view32_return() == "utf32 secret 🎂" if hasattr(m, "has_u8string"): - assert m.string_view8_return() == u"utf8 secret 🎂" + assert m.string_view8_return() == "utf8 secret 🎂" with capture: m.string_view_print("Hi") m.string_view_print("utf8 🎂") - m.string_view16_print(u"utf16 🎂") - m.string_view32_print(u"utf32 🎂") + m.string_view16_print("utf16 🎂") + m.string_view32_print("utf32 🎂") assert ( capture - == u""" + == """ Hi 2 utf8 🎂 9 utf16 🎂 8 @@ -172,10 +176,10 @@ def test_string_view(capture): if hasattr(m, "has_u8string"): with capture: m.string_view8_print("Hi") - m.string_view8_print(u"utf8 🎂") + m.string_view8_print("utf8 🎂") assert ( capture - == u""" + == """ Hi 2 utf8 🎂 9 """ @@ -184,11 +188,11 @@ def test_string_view(capture): with capture: m.string_view_print("Hi, ascii") m.string_view_print("Hi, utf8 🎂") - m.string_view16_print(u"Hi, utf16 🎂") - m.string_view32_print(u"Hi, utf32 🎂") + m.string_view16_print("Hi, utf16 🎂") + m.string_view32_print("Hi, utf32 🎂") assert ( capture - == u""" + == """ Hi, ascii 9 Hi, utf8 🎂 13 Hi, utf16 🎂 12 @@ -198,15 +202,25 @@ def test_string_view(capture): if hasattr(m, "has_u8string"): with capture: m.string_view8_print("Hi, ascii") - m.string_view8_print(u"Hi, utf8 🎂") + m.string_view8_print("Hi, utf8 🎂") assert ( capture - == u""" + == """ Hi, ascii 9 Hi, utf8 🎂 13 """ ) + assert m.string_view_bytes() == b"abc \x80\x80 def" + assert m.string_view_str() == "abc ‽ def" + assert m.string_view_from_bytes("abc ‽ def".encode()) == "abc ‽ def" + if hasattr(m, "has_u8string"): + assert m.string_view8_str() == "abc ‽ def" + assert m.string_view_memoryview() == "Have some 🎂".encode() + + assert m.bytes_from_type_with_both_operator_string_and_string_view() == b"success" + assert m.str_from_type_with_both_operator_string_and_string_view() == "success" + def test_integer_casting(): """Issue #929 - out-of-range integer values shouldn't be accepted""" @@ -214,20 +228,8 @@ def test_integer_casting(): assert m.i64_str(-1) == "-1" assert m.i32_str(2000000000) == "2000000000" assert m.u32_str(2000000000) == "2000000000" - if env.PY2: - assert m.i32_str(long(-1)) == "-1" # noqa: F821 undefined name 'long' - assert m.i64_str(long(-1)) == "-1" # noqa: F821 undefined name 'long' - assert ( - m.i64_str(long(-999999999999)) # noqa: F821 undefined name 'long' - == "-999999999999" - ) - assert ( - m.u64_str(long(999999999999)) # noqa: F821 undefined name 'long' - == "999999999999" - ) - else: - assert m.i64_str(-999999999999) == "-999999999999" - assert m.u64_str(999999999999) == "999999999999" + assert m.i64_str(-999999999999) == "-999999999999" + assert m.u64_str(999999999999) == "999999999999" with pytest.raises(TypeError) as excinfo: m.u32_str(-1) @@ -242,46 +244,38 @@ def test_integer_casting(): m.i32_str(3000000000) assert "incompatible function arguments" in str(excinfo.value) - if env.PY2: - with pytest.raises(TypeError) as excinfo: - m.u32_str(long(-1)) # noqa: F821 undefined name 'long' - assert "incompatible function arguments" in str(excinfo.value) - with pytest.raises(TypeError) as excinfo: - m.u64_str(long(-1)) # noqa: F821 undefined name 'long' - assert "incompatible function arguments" in str(excinfo.value) - def test_int_convert(): - class Int(object): + class Int: def __int__(self): return 42 - class NotInt(object): + class NotInt: pass - class Float(object): + class Float: def __float__(self): return 41.99999 - class Index(object): + class Index: def __index__(self): return 42 - class IntAndIndex(object): + class IntAndIndex: def __int__(self): return 42 def __index__(self): return 0 - class RaisingTypeErrorOnIndex(object): + class RaisingTypeErrorOnIndex: def __index__(self): raise TypeError def __int__(self): return 42 - class RaisingValueErrorOnIndex(object): + class RaisingValueErrorOnIndex: def __index__(self): raise ValueError @@ -299,7 +293,13 @@ def test_int_convert(): assert convert(7) == 7 assert noconvert(7) == 7 cant_convert(3.14159) - assert convert(Int()) == 42 + # TODO: Avoid DeprecationWarning in `PyLong_AsLong` (and similar) + # TODO: PyPy 3.8 does not behave like CPython 3.8 here yet (7.3.7) + if (3, 8) <= sys.version_info < (3, 10) and env.CPYTHON: + with env.deprecated_call(): + assert convert(Int()) == 42 + else: + assert convert(Int()) == 42 requires_conversion(Int()) cant_convert(NotInt()) cant_convert(Float()) @@ -329,7 +329,14 @@ def test_numpy_int_convert(): assert noconvert(np.intc(42)) == 42 # The implicit conversion from np.float32 is undesirable but currently accepted. - assert convert(np.float32(3.14159)) == 3 + # TODO: Avoid DeprecationWarning in `PyLong_AsLong` (and similar) + # TODO: PyPy 3.8 does not behave like CPython 3.8 here yet (7.3.7) + # https://github.com/pybind/pybind11/issues/3408 + if (3, 8) <= sys.version_info < (3, 10) and env.CPYTHON: + with env.deprecated_call(): + assert convert(np.float32(3.14159)) == 3 + else: + assert convert(np.float32(3.14159)) == 3 require_implicit(np.float32(3.14159)) @@ -414,13 +421,15 @@ def test_reference_wrapper(): a2 = m.refwrap_list(copy=True) assert [x.value for x in a1] == [2, 3] assert [x.value for x in a2] == [2, 3] - assert not a1[0] is a2[0] and not a1[1] is a2[1] + assert a1[0] is not a2[0] + assert a1[1] is not a2[1] b1 = m.refwrap_list(copy=False) b2 = m.refwrap_list(copy=False) assert [x.value for x in b1] == [1, 2] assert [x.value for x in b2] == [1, 2] - assert b1[0] is b2[0] and b1[1] is b2[1] + assert b1[0] is b2[0] + assert b1[1] is b2[1] assert m.refwrap_iiw(IncType(5)) == 5 assert m.refwrap_call_iiw(IncType(10), m.refwrap_iiw) == [10, 10, 10, 10] @@ -452,7 +461,7 @@ def test_bool_caster(): require_implicit(None) assert convert(None) is False - class A(object): + class A: def __init__(self, x): self.x = x @@ -462,7 +471,7 @@ def test_bool_caster(): def __bool__(self): return self.x - class B(object): + class B: pass # Arbitrary objects are not accepted @@ -492,17 +501,9 @@ def test_numpy_bool(): def test_int_long(): - """In Python 2, a C++ int should return a Python int rather than long - if possible: longs are not always accepted where ints are used (such - as the argument to sys.exit()). A C++ long long is always a Python - long.""" - - import sys - - must_be_long = type(getattr(sys, "maxint", 1) + 1) assert isinstance(m.int_cast(), int) assert isinstance(m.long_cast(), int) - assert isinstance(m.longlong_cast(), must_be_long) + assert isinstance(m.longlong_cast(), int) def test_void_caster_2(): @@ -511,7 +512,7 @@ def test_void_caster_2(): def test_const_ref_caster(): """Verifies that const-ref is propagated through type_caster cast_op. - The returned ConstRefCasted type is a mimimal type that is constructed to + The returned ConstRefCasted type is a minimal type that is constructed to reference the casting mode used. """ x = False diff --git a/tests/test_call_policies.cpp b/tests/test_call_policies.cpp index 26c83f81..d177008c 100644 --- a/tests/test_call_policies.cpp +++ b/tests/test_call_policies.cpp @@ -40,17 +40,17 @@ TEST_SUBMODULE(call_policies, m) { Child(Child &&) = default; ~Child() { py::print("Releasing child."); } }; - py::class_<Child>(m, "Child") - .def(py::init<>()); + py::class_<Child>(m, "Child").def(py::init<>()); class Parent { public: Parent() { py::print("Allocating parent."); } - Parent(const Parent& parent) = default; + Parent(const Parent &parent) = default; ~Parent() { py::print("Releasing parent."); } - void addChild(Child *) { } + void addChild(Child *) {} Child *returnChild() { return new Child(); } Child *returnNullChild() { return nullptr; } + static Child *staticFunction(Parent *) { return new Child(); } }; py::class_<Parent>(m, "Parent") .def(py::init<>()) @@ -60,7 +60,13 @@ TEST_SUBMODULE(call_policies, m) { .def("returnChild", &Parent::returnChild) .def("returnChildKeepAlive", &Parent::returnChild, py::keep_alive<1, 0>()) .def("returnNullChildKeepAliveChild", &Parent::returnNullChild, py::keep_alive<1, 0>()) - .def("returnNullChildKeepAliveParent", &Parent::returnNullChild, py::keep_alive<0, 1>()); + .def("returnNullChildKeepAliveParent", &Parent::returnNullChild, py::keep_alive<0, 1>()) + .def_static("staticFunction", &Parent::staticFunction, py::keep_alive<1, 0>()); + + m.def( + "free_function", [](Parent *, Child *) {}, py::keep_alive<1, 2>()); + m.def( + "invalid_arg_index", [] {}, py::keep_alive<0, 1>()); #if !defined(PYPY_VERSION) // test_alive_gc @@ -68,29 +74,37 @@ TEST_SUBMODULE(call_policies, m) { public: using Parent::Parent; }; - py::class_<ParentGC, Parent>(m, "ParentGC", py::dynamic_attr()) - .def(py::init<>()); + py::class_<ParentGC, Parent>(m, "ParentGC", py::dynamic_attr()).def(py::init<>()); #endif // test_call_guard m.def("unguarded_call", &CustomGuard::report_status); m.def("guarded_call", &CustomGuard::report_status, py::call_guard<CustomGuard>()); - m.def("multiple_guards_correct_order", []() { - return CustomGuard::report_status() + std::string(" & ") + DependentGuard::report_status(); - }, py::call_guard<CustomGuard, DependentGuard>()); - - m.def("multiple_guards_wrong_order", []() { - return DependentGuard::report_status() + std::string(" & ") + CustomGuard::report_status(); - }, py::call_guard<DependentGuard, CustomGuard>()); + m.def( + "multiple_guards_correct_order", + []() { + return CustomGuard::report_status() + std::string(" & ") + + DependentGuard::report_status(); + }, + py::call_guard<CustomGuard, DependentGuard>()); + + m.def( + "multiple_guards_wrong_order", + []() { + return DependentGuard::report_status() + std::string(" & ") + + CustomGuard::report_status(); + }, + py::call_guard<DependentGuard, CustomGuard>()); #if defined(WITH_THREAD) && !defined(PYPY_VERSION) // `py::call_guard<py::gil_scoped_release>()` should work in PyPy as well, // but it's unclear how to test it without `PyGILState_GetThisThreadState`. auto report_gil_status = []() { auto is_gil_held = false; - if (auto tstate = py::detail::get_thread_state_unchecked()) + if (auto *tstate = py::detail::get_thread_state_unchecked()) { is_gil_held = (tstate == PyGILState_GetThisThreadState()); + } return is_gil_held ? "GIL held" : "GIL released"; }; diff --git a/tests/test_call_policies.py b/tests/test_call_policies.py index e0413d14..61605641 100644 --- a/tests/test_call_policies.py +++ b/tests/test_call_policies.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- import pytest import env # noqa: F401 - -from pybind11_tests import call_policies as m from pybind11_tests import ConstructorStats +from pybind11_tests import call_policies as m @pytest.mark.xfail("env.PYPY", reason="sometimes comes out 1 off on PyPy", strict=False) @@ -46,6 +44,19 @@ def test_keep_alive_argument(capture): """ ) + p = m.Parent() + c = m.Child() + assert ConstructorStats.detail_reg_inst() == n_inst + 2 + m.free_function(p, c) + del c + assert ConstructorStats.detail_reg_inst() == n_inst + 2 + del p + assert ConstructorStats.detail_reg_inst() == n_inst + + with pytest.raises(RuntimeError) as excinfo: + m.invalid_arg_index() + assert str(excinfo.value) == "Could not activate keep_alive!" + def test_keep_alive_return_value(capture): n_inst = ConstructorStats.detail_reg_inst() @@ -85,6 +96,23 @@ def test_keep_alive_return_value(capture): """ ) + p = m.Parent() + assert ConstructorStats.detail_reg_inst() == n_inst + 1 + with capture: + m.Parent.staticFunction(p) + assert ConstructorStats.detail_reg_inst() == n_inst + 2 + assert capture == "Allocating child." + with capture: + del p + assert ConstructorStats.detail_reg_inst() == n_inst + assert ( + capture + == """ + Releasing parent. + Releasing child. + """ + ) + # https://foss.heptapod.net/pypy/pypy/-/issues/2447 @pytest.mark.xfail("env.PYPY", reason="_PyObject_GetDictPtr is unimplemented") diff --git a/tests/test_callbacks.cpp b/tests/test_callbacks.cpp index dffe538f..2fd05dec 100644 --- a/tests/test_callbacks.cpp +++ b/tests/test_callbacks.cpp @@ -7,72 +7,79 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/functional.h> -#include <thread> +#include "constructor_stats.h" +#include "pybind11_tests.h" + +#include <thread> int dummy_function(int i) { return i + 1; } TEST_SUBMODULE(callbacks, m) { // test_callbacks, test_function_signatures - m.def("test_callback1", [](py::object func) { return func(); }); - m.def("test_callback2", [](py::object func) { return func("Hello", 'x', true, 5); }); + m.def("test_callback1", [](const py::object &func) { return func(); }); + m.def("test_callback2", [](const py::object &func) { return func("Hello", 'x', true, 5); }); m.def("test_callback3", [](const std::function<int(int)> &func) { - return "func(43) = " + std::to_string(func(43)); }); - m.def("test_callback4", []() -> std::function<int(int)> { return [](int i) { return i+1; }; }); - m.def("test_callback5", []() { - return py::cpp_function([](int i) { return i+1; }, py::arg("number")); + return "func(43) = " + std::to_string(func(43)); }); + m.def("test_callback4", + []() -> std::function<int(int)> { return [](int i) { return i + 1; }; }); + m.def("test_callback5", + []() { return py::cpp_function([](int i) { return i + 1; }, py::arg("number")); }); // test_keyword_args_and_generalized_unpacking - m.def("test_tuple_unpacking", [](py::function f) { + m.def("test_tuple_unpacking", [](const py::function &f) { auto t1 = py::make_tuple(2, 3); auto t2 = py::make_tuple(5, 6); return f("positional", 1, *t1, 4, *t2); }); - m.def("test_dict_unpacking", [](py::function f) { - auto d1 = py::dict("key"_a="value", "a"_a=1); + m.def("test_dict_unpacking", [](const py::function &f) { + auto d1 = py::dict("key"_a = "value", "a"_a = 1); auto d2 = py::dict(); - auto d3 = py::dict("b"_a=2); + auto d3 = py::dict("b"_a = 2); return f("positional", 1, **d1, **d2, **d3); }); - m.def("test_keyword_args", [](py::function f) { - return f("x"_a=10, "y"_a=20); - }); + m.def("test_keyword_args", [](const py::function &f) { return f("x"_a = 10, "y"_a = 20); }); - m.def("test_unpacking_and_keywords1", [](py::function f) { + m.def("test_unpacking_and_keywords1", [](const py::function &f) { auto args = py::make_tuple(2); - auto kwargs = py::dict("d"_a=4); - return f(1, *args, "c"_a=3, **kwargs); + auto kwargs = py::dict("d"_a = 4); + return f(1, *args, "c"_a = 3, **kwargs); }); - m.def("test_unpacking_and_keywords2", [](py::function f) { - auto kwargs1 = py::dict("a"_a=1); - auto kwargs2 = py::dict("c"_a=3, "d"_a=4); - return f("positional", *py::make_tuple(1), 2, *py::make_tuple(3, 4), 5, - "key"_a="value", **kwargs1, "b"_a=2, **kwargs2, "e"_a=5); + m.def("test_unpacking_and_keywords2", [](const py::function &f) { + auto kwargs1 = py::dict("a"_a = 1); + auto kwargs2 = py::dict("c"_a = 3, "d"_a = 4); + return f("positional", + *py::make_tuple(1), + 2, + *py::make_tuple(3, 4), + 5, + "key"_a = "value", + **kwargs1, + "b"_a = 2, + **kwargs2, + "e"_a = 5); }); - m.def("test_unpacking_error1", [](py::function f) { - auto kwargs = py::dict("x"_a=3); - return f("x"_a=1, "y"_a=2, **kwargs); // duplicate ** after keyword + m.def("test_unpacking_error1", [](const py::function &f) { + auto kwargs = py::dict("x"_a = 3); + return f("x"_a = 1, "y"_a = 2, **kwargs); // duplicate ** after keyword }); - m.def("test_unpacking_error2", [](py::function f) { - auto kwargs = py::dict("x"_a=3); - return f(**kwargs, "x"_a=1); // duplicate keyword after ** + m.def("test_unpacking_error2", [](const py::function &f) { + auto kwargs = py::dict("x"_a = 3); + return f(**kwargs, "x"_a = 1); // duplicate keyword after ** }); - m.def("test_arg_conversion_error1", [](py::function f) { - f(234, UnregisteredType(), "kw"_a=567); - }); + m.def("test_arg_conversion_error1", + [](const py::function &f) { f(234, UnregisteredType(), "kw"_a = 567); }); - m.def("test_arg_conversion_error2", [](py::function f) { - f(234, "expected_name"_a=UnregisteredType(), "kw"_a=567); + m.def("test_arg_conversion_error2", [](const py::function &f) { + f(234, "expected_name"_a = UnregisteredType(), "kw"_a = 567); }); // test_lambda_closure_cleanup @@ -80,41 +87,87 @@ TEST_SUBMODULE(callbacks, m) { Payload() { print_default_created(this); } ~Payload() { print_destroyed(this); } Payload(const Payload &) { print_copy_created(this); } - Payload(Payload &&) { print_move_created(this); } + Payload(Payload &&) noexcept { print_move_created(this); } }; // Export the payload constructor statistics for testing purposes: m.def("payload_cstats", &ConstructorStats::get<Payload>); - /* Test cleanup of lambda closure */ - m.def("test_cleanup", []() -> std::function<void(void)> { + m.def("test_lambda_closure_cleanup", []() -> std::function<void()> { Payload p; + // In this situation, `Func` in the implementation of + // `cpp_function::initialize` is NOT trivially destructible. return [p]() { /* p should be cleaned up when the returned function is garbage collected */ (void) p; }; }); + class CppCallable { + public: + CppCallable() { track_default_created(this); } + ~CppCallable() { track_destroyed(this); } + CppCallable(const CppCallable &) { track_copy_created(this); } + CppCallable(CppCallable &&) noexcept { track_move_created(this); } + void operator()() {} + }; + + m.def("test_cpp_callable_cleanup", []() { + // Related issue: https://github.com/pybind/pybind11/issues/3228 + // Related PR: https://github.com/pybind/pybind11/pull/3229 + py::list alive_counts; + ConstructorStats &stat = ConstructorStats::get<CppCallable>(); + alive_counts.append(stat.alive()); + { + CppCallable cpp_callable; + alive_counts.append(stat.alive()); + { + // In this situation, `Func` in the implementation of + // `cpp_function::initialize` IS trivially destructible, + // only `capture` is not. + py::cpp_function py_func(cpp_callable); + py::detail::silence_unused_warnings(py_func); + alive_counts.append(stat.alive()); + } + alive_counts.append(stat.alive()); + { + py::cpp_function py_func(std::move(cpp_callable)); + py::detail::silence_unused_warnings(py_func); + alive_counts.append(stat.alive()); + } + alive_counts.append(stat.alive()); + } + alive_counts.append(stat.alive()); + return alive_counts; + }); + // test_cpp_function_roundtrip /* Test if passing a function pointer from C++ -> Python -> C++ yields the original pointer */ m.def("dummy_function", &dummy_function); + m.def("dummy_function_overloaded", [](int i, int j) { return i + j; }); + m.def("dummy_function_overloaded", &dummy_function); m.def("dummy_function2", [](int i, int j) { return i + j; }); - m.def("roundtrip", [](std::function<int(int)> f, bool expect_none = false) { - if (expect_none && f) - throw std::runtime_error("Expected None to be converted to empty std::function"); - return f; - }, py::arg("f"), py::arg("expect_none")=false); + m.def( + "roundtrip", + [](std::function<int(int)> f, bool expect_none = false) { + if (expect_none && f) { + throw std::runtime_error("Expected None to be converted to empty std::function"); + } + return f; + }, + py::arg("f"), + py::arg("expect_none") = false); m.def("test_dummy_function", [](const std::function<int(int)> &f) -> std::string { using fn_type = int (*)(int); - auto result = f.target<fn_type>(); + const auto *result = f.target<fn_type>(); if (!result) { auto r = f(1); return "can't convert to function pointer: eval(1) = " + std::to_string(r); - } else if (*result == dummy_function) { + } + if (*result == dummy_function) { auto r = (*result)(1); return "matches dummy_function: eval(1) = " + std::to_string(r); - } else { - return "argument does NOT match dummy_function. This should never happen!"; } + return "argument does NOT match dummy_function. This should never happen!"; }); class AbstractBase { @@ -122,10 +175,11 @@ TEST_SUBMODULE(callbacks, m) { // [workaround(intel)] = default does not work here // Defaulting this destructor results in linking errors with the Intel compiler // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) - virtual ~AbstractBase() {}; // NOLINT(modernize-use-equals-default) + virtual ~AbstractBase() {} // NOLINT(modernize-use-equals-default) virtual unsigned int func() = 0; }; - m.def("func_accepting_func_accepting_base", [](std::function<double(AbstractBase&)>) { }); + m.def("func_accepting_func_accepting_base", + [](const std::function<double(AbstractBase &)> &) {}); struct MovableObject { bool valid = true; @@ -133,8 +187,8 @@ TEST_SUBMODULE(callbacks, m) { MovableObject() = default; MovableObject(const MovableObject &) = default; MovableObject &operator=(const MovableObject &) = default; - MovableObject(MovableObject &&o) : valid(o.valid) { o.valid = false; } - MovableObject &operator=(MovableObject &&o) { + MovableObject(MovableObject &&o) noexcept : valid(o.valid) { o.valid = false; } + MovableObject &operator=(MovableObject &&o) noexcept { valid = o.valid; o.valid = false; return *this; @@ -143,9 +197,9 @@ TEST_SUBMODULE(callbacks, m) { py::class_<MovableObject>(m, "MovableObject"); // test_movable_object - m.def("callback_with_movable", [](std::function<void(MovableObject &)> f) { + m.def("callback_with_movable", [](const std::function<void(MovableObject &)> &f) { auto x = MovableObject(); - f(x); // lvalue reference shouldn't move out object + f(x); // lvalue reference shouldn't move out object return x.valid; // must still return `true` }); @@ -155,9 +209,16 @@ TEST_SUBMODULE(callbacks, m) { .def(py::init<>()) .def("triple", [](CppBoundMethodTest &, int val) { return 3 * val; }); + // This checks that builtin functions can be passed as callbacks + // rather than throwing RuntimeError due to trying to extract as capsule + m.def("test_sum_builtin", + [](const std::function<double(py::iterable)> &sum_builtin, const py::iterable &i) { + return sum_builtin(i); + }); + // test async Python callbacks using callback_f = std::function<void(int)>; - m.def("test_async_callback", [](callback_f f, py::list work) { + m.def("test_async_callback", [](const callback_f &f, const py::list &work) { // make detached thread that calls `f` with piece of work after a little delay auto start_f = [f](int j) { auto invoke_f = [f, j] { @@ -169,7 +230,51 @@ TEST_SUBMODULE(callbacks, m) { }; // spawn worker threads - for (auto i : work) + for (auto i : work) { start_f(py::cast<int>(i)); + } + }); + + m.def("callback_num_times", [](const py::function &f, std::size_t num) { + for (std::size_t i = 0; i < num; i++) { + f(); + } }); + + auto *custom_def = []() { + static PyMethodDef def; + def.ml_name = "example_name"; + def.ml_doc = "Example doc"; + def.ml_meth = [](PyObject *, PyObject *args) -> PyObject * { + if (PyTuple_Size(args) != 1) { + throw std::runtime_error("Invalid number of arguments for example_name"); + } + PyObject *first = PyTuple_GetItem(args, 0); + if (!PyLong_Check(first)) { + throw std::runtime_error("Invalid argument to example_name"); + } + auto result = py::cast(PyLong_AsLong(first) * 9); + return result.release().ptr(); + }; + def.ml_flags = METH_VARARGS; + return &def; + }(); + + // rec_capsule with name that has the same value (but not pointer) as our internal one + // This capsule should be detected by our code as foreign and not inspected as the pointers + // shouldn't match + constexpr const char *rec_capsule_name + = pybind11::detail::internals_function_record_capsule_name; + py::capsule rec_capsule(std::malloc(1), [](void *data) { std::free(data); }); + rec_capsule.set_name(rec_capsule_name); + m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr())); + + // This test requires a new ABI version to pass +#if PYBIND11_INTERNALS_VERSION > 4 + // rec_capsule with nullptr name + py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); }); + m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr())); +#else + m.add_object("custom_function2", py::none()); +#endif } diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py index 039b877c..4a652f53 100644 --- a/tests/test_callbacks.py +++ b/tests/test_callbacks.py @@ -1,7 +1,11 @@ -# -*- coding: utf-8 -*- +import time +from threading import Thread + import pytest + +import env # noqa: F401 from pybind11_tests import callbacks as m -from threading import Thread +from pybind11_tests import detailed_error_messages_enabled def test_callbacks(): @@ -14,7 +18,7 @@ def test_callbacks(): return "func2", a, b, c, d def func3(a): - return "func3({})".format(a) + return f"func3({a})" assert m.test_callback1(func1) == "func1" assert m.test_callback2(func2) == ("func2", "Hello", "x", True, 5) @@ -67,21 +71,35 @@ def test_keyword_args_and_generalized_unpacking(): with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error1(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'1' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error2(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'expected_name' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'expected_name' to Python object " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) def test_lambda_closure_cleanup(): - m.test_cleanup() + m.test_lambda_closure_cleanup() cstats = m.payload_cstats() assert cstats.alive() == 0 assert cstats.copy_constructions == 1 assert cstats.move_constructions >= 1 +def test_cpp_callable_cleanup(): + alive_counts = m.test_cpp_callable_cleanup() + assert alive_counts == [0, 1, 2, 1, 2, 1, 0] + + def test_cpp_function_roundtrip(): """Test if passing a function pointer from C++ -> Python -> C++ yields the original pointer""" @@ -92,6 +110,10 @@ def test_cpp_function_roundtrip(): m.test_dummy_function(m.roundtrip(m.dummy_function)) == "matches dummy_function: eval(1) = 2" ) + assert ( + m.test_dummy_function(m.dummy_function_overloaded) + == "matches dummy_function: eval(1) = 2" + ) assert m.roundtrip(None, expect_none=True) is None assert ( m.test_dummy_function(lambda x: x + 2) @@ -119,6 +141,16 @@ def test_movable_object(): assert m.callback_with_movable(lambda _: None) is True +@pytest.mark.skipif( + "env.PYPY", + reason="PyPy segfaults on here. See discussion on #1413.", +) +def test_python_builtins(): + """Test if python builtins like sum() can be used as callbacks""" + assert m.test_sum_builtin(sum, [1, 2, 3]) == 6 + assert m.test_sum_builtin(sum, []) == 0 + + def test_async_callbacks(): # serves as state for async callback class Item: @@ -139,10 +171,48 @@ def test_async_callbacks(): from time import sleep sleep(0.5) - assert sum(res) == sum([x + 3 for x in work]) + assert sum(res) == sum(x + 3 for x in work) def test_async_async_callbacks(): t = Thread(target=test_async_callbacks) t.start() t.join() + + +def test_callback_num_times(): + # Super-simple micro-benchmarking related to PR #2919. + # Example runtimes (Intel Xeon 2.2GHz, fully optimized): + # num_millions 1, repeats 2: 0.1 secs + # num_millions 20, repeats 10: 11.5 secs + one_million = 1000000 + num_millions = 1 # Try 20 for actual micro-benchmarking. + repeats = 2 # Try 10. + rates = [] + for rep in range(repeats): + t0 = time.time() + m.callback_num_times(lambda: None, num_millions * one_million) + td = time.time() - t0 + rate = num_millions / td if td else 0 + rates.append(rate) + if not rep: + print() + print( + f"callback_num_times: {num_millions:d} million / {td:.3f} seconds = {rate:.3f} million / second" + ) + if len(rates) > 1: + print("Min Mean Max") + print(f"{min(rates):6.3f} {sum(rates) / len(rates):6.3f} {max(rates):6.3f}") + + +def test_custom_func(): + assert m.custom_function(4) == 36 + assert m.roundtrip(m.custom_function)(4) == 36 + + +@pytest.mark.skipif( + m.custom_function2 is None, reason="Current PYBIND11_INTERNALS_VERSION too low" +) +def test_custom_func2(): + assert m.custom_function2(3) == 27 + assert m.roundtrip(m.custom_function2)(3) == 27 diff --git a/tests/test_chrono.cpp b/tests/test_chrono.cpp index 65370508..8be0ffd1 100644 --- a/tests/test_chrono.cpp +++ b/tests/test_chrono.cpp @@ -8,21 +8,20 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/chrono.h> + +#include "pybind11_tests.h" + #include <chrono> struct different_resolutions { - using time_point_h = std::chrono::time_point< - std::chrono::system_clock, std::chrono::hours>; - using time_point_m = std::chrono::time_point< - std::chrono::system_clock, std::chrono::minutes>; - using time_point_s = std::chrono::time_point< - std::chrono::system_clock, std::chrono::seconds>; - using time_point_ms = std::chrono::time_point< - std::chrono::system_clock, std::chrono::milliseconds>; - using time_point_us = std::chrono::time_point< - std::chrono::system_clock, std::chrono::microseconds>; + using time_point_h = std::chrono::time_point<std::chrono::system_clock, std::chrono::hours>; + using time_point_m = std::chrono::time_point<std::chrono::system_clock, std::chrono::minutes>; + using time_point_s = std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>; + using time_point_ms + = std::chrono::time_point<std::chrono::system_clock, std::chrono::milliseconds>; + using time_point_us + = std::chrono::time_point<std::chrono::system_clock, std::chrono::microseconds>; time_point_h timestamp_h; time_point_m timestamp_m; time_point_s timestamp_s; @@ -65,12 +64,11 @@ TEST_SUBMODULE(chrono, m) { // Roundtrip a duration in microseconds from a float argument m.def("test_chrono7", [](std::chrono::microseconds t) { return t; }); // Float durations (issue #719) - m.def("test_chrono_float_diff", [](std::chrono::duration<float> a, std::chrono::duration<float> b) { - return a - b; }); + m.def("test_chrono_float_diff", + [](std::chrono::duration<float> a, std::chrono::duration<float> b) { return a - b; }); - m.def("test_nano_timepoint", [](timestamp start, timespan delta) -> timestamp { - return start + delta; - }); + m.def("test_nano_timepoint", + [](timestamp start, timespan delta) -> timestamp { return start + delta; }); // Test different resolutions py::class_<different_resolutions>(m, "different_resolutions") @@ -79,6 +77,5 @@ TEST_SUBMODULE(chrono, m) { .def_readwrite("timestamp_m", &different_resolutions::timestamp_m) .def_readwrite("timestamp_s", &different_resolutions::timestamp_s) .def_readwrite("timestamp_ms", &different_resolutions::timestamp_ms) - .def_readwrite("timestamp_us", &different_resolutions::timestamp_us) - ; + .def_readwrite("timestamp_us", &different_resolutions::timestamp_us); } diff --git a/tests/test_chrono.py b/tests/test_chrono.py index e9e24e08..a29316c3 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- -from pybind11_tests import chrono as m import datetime + import pytest import env # noqa: F401 +from pybind11_tests import chrono as m def test_chrono_system_clock(): - # Get the time from both c++ and datetime date0 = datetime.datetime.today() date1 = m.test_chrono1() @@ -39,9 +38,7 @@ def test_chrono_system_clock_roundtrip(): # They should be identical (no information lost on roundtrip) diff = abs(date1 - date2) - assert diff.days == 0 - assert diff.seconds == 0 - assert diff.microseconds == 0 + assert diff == datetime.timedelta(0) def test_chrono_system_clock_roundtrip_date(): @@ -64,9 +61,7 @@ def test_chrono_system_clock_roundtrip_date(): assert diff.microseconds == 0 # Year, Month & Day should be the same after the round trip - assert date1.year == date2.year - assert date1.month == date2.month - assert date1.day == date2.day + assert date1 == date2 # There should be no time information assert time2.hour == 0 @@ -104,7 +99,7 @@ SKIP_TZ_ENV_ON_WIN = pytest.mark.skipif( ) def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): if tz is not None: - monkeypatch.setenv("TZ", "/usr/share/zoneinfo/{}".format(tz)) + monkeypatch.setenv("TZ", f"/usr/share/zoneinfo/{tz}") # Roundtrip the time datetime2 = m.test_chrono2(time1) @@ -117,10 +112,7 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): assert isinstance(time2, datetime.time) # Hour, Minute, Second & Microsecond should be the same after the round trip - assert time1.hour == time2.hour - assert time1.minute == time2.minute - assert time1.second == time2.second - assert time1.microsecond == time2.microsecond + assert time1 == time2 # There should be no date information (i.e. date = python base date) assert date2.year == 1970 @@ -129,7 +121,6 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): def test_chrono_duration_roundtrip(): - # Get the difference between two times (a timedelta) date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -140,35 +131,33 @@ def test_chrono_duration_roundtrip(): cpp_diff = m.test_chrono3(diff) - assert cpp_diff.days == diff.days - assert cpp_diff.seconds == diff.seconds - assert cpp_diff.microseconds == diff.microseconds + assert cpp_diff == diff + # Negative timedelta roundtrip + diff = datetime.timedelta(microseconds=-1) + cpp_diff = m.test_chrono3(diff) -def test_chrono_duration_subtraction_equivalence(): + assert cpp_diff == diff + +def test_chrono_duration_subtraction_equivalence(): date1 = datetime.datetime.today() date2 = datetime.datetime.today() diff = date2 - date1 cpp_diff = m.test_chrono4(date2, date1) - assert cpp_diff.days == diff.days - assert cpp_diff.seconds == diff.seconds - assert cpp_diff.microseconds == diff.microseconds + assert cpp_diff == diff def test_chrono_duration_subtraction_equivalence_date(): - date1 = datetime.date.today() date2 = datetime.date.today() diff = date2 - date1 cpp_diff = m.test_chrono4(date2, date1) - assert cpp_diff.days == diff.days - assert cpp_diff.seconds == diff.seconds - assert cpp_diff.microseconds == diff.microseconds + assert cpp_diff == diff def test_chrono_steady_clock(): @@ -183,9 +172,7 @@ def test_chrono_steady_clock_roundtrip(): assert isinstance(time2, datetime.timedelta) # They should be identical (no information lost on roundtrip) - assert time1.days == time2.days - assert time1.seconds == time2.seconds - assert time1.microseconds == time2.microseconds + assert time1 == time2 def test_floating_point_duration(): diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 6ce928ca..7241bc88 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -11,28 +11,52 @@ // Intel compiler requires a separate header file to support aligned new operators // and does not set the __cpp_aligned_new feature macro. // This header needs to be included before pybind11. -#include <aligned_new> +# include <aligned_new> #endif -#include "pybind11_tests.h" +#include <pybind11/stl.h> + #include "constructor_stats.h" #include "local_bindings.h" -#include <pybind11/stl.h> +#include "pybind11_tests.h" -#if defined(_MSC_VER) -# pragma warning(disable: 4324) // warning C4324: structure was padded due to alignment specifier -#endif +#include <utility> + +PYBIND11_WARNING_DISABLE_MSVC(4324) +// warning C4324: structure was padded due to alignment specifier // test_brace_initialization struct NoBraceInitialization { - NoBraceInitialization(std::vector<int> v) : vec{std::move(v)} {} + explicit NoBraceInitialization(std::vector<int> v) : vec{std::move(v)} {} template <typename T> NoBraceInitialization(std::initializer_list<T> l) : vec(l) {} std::vector<int> vec; }; +namespace test_class { +namespace pr4220_tripped_over_this { // PR #4227 + +template <int> +struct SoEmpty {}; + +template <typename T> +std::string get_msg(const T &) { + return "This is really only meant to exercise successful compilation."; +} + +using Empty0 = SoEmpty<0x0>; + +void bind_empty0(py::module_ &m) { + py::class_<Empty0>(m, "Empty0").def(py::init<>()).def("get_msg", get_msg<Empty0>); +} + +} // namespace pr4220_tripped_over_this +} // namespace test_class + TEST_SUBMODULE(class_, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + // test_instance struct NoConstructor { NoConstructor() = default; @@ -45,10 +69,26 @@ TEST_SUBMODULE(class_, m) { } ~NoConstructor() { print_destroyed(this); } }; + struct NoConstructorNew { + NoConstructorNew() = default; + NoConstructorNew(const NoConstructorNew &) = default; + NoConstructorNew(NoConstructorNew &&) = default; + static NoConstructorNew *new_instance() { + auto *ptr = new NoConstructorNew(); + print_created(ptr, "via new_instance"); + return ptr; + } + ~NoConstructorNew() { print_destroyed(this); } + }; py::class_<NoConstructor>(m, "NoConstructor") .def_static("new_instance", &NoConstructor::new_instance, "Return an instance"); + py::class_<NoConstructorNew>(m, "NoConstructorNew") + .def(py::init([]() { return nullptr; })) // Need a NOOP __init__ + .def_static("__new__", + [](const py::object &) { return NoConstructorNew::new_instance(); }); + // test_inheritance class Pet { public: @@ -56,6 +96,7 @@ TEST_SUBMODULE(class_, m) { : m_name(name), m_species(species) {} std::string name() const { return m_name; } std::string species() const { return m_species; } + private: std::string m_name; std::string m_species; @@ -63,18 +104,18 @@ TEST_SUBMODULE(class_, m) { class Dog : public Pet { public: - Dog(const std::string &name) : Pet(name, "dog") {} + explicit Dog(const std::string &name) : Pet(name, "dog") {} std::string bark() const { return "Woof!"; } }; class Rabbit : public Pet { public: - Rabbit(const std::string &name) : Pet(name, "parrot") {} + explicit Rabbit(const std::string &name) : Pet(name, "parrot") {} }; class Hamster : public Pet { public: - Hamster(const std::string &name) : Pet(name, "rodent") {} + explicit Hamster(const std::string &name) : Pet(name, "rodent") {} }; class Chimera : public Pet { @@ -82,27 +123,24 @@ TEST_SUBMODULE(class_, m) { }; py::class_<Pet> pet_class(m, "Pet"); - pet_class - .def(py::init<std::string, std::string>()) + pet_class.def(py::init<std::string, std::string>()) .def("name", &Pet::name) .def("species", &Pet::species); /* One way of declaring a subclass relationship: reference parent's class_ object */ - py::class_<Dog>(m, "Dog", pet_class) - .def(py::init<std::string>()); + py::class_<Dog>(m, "Dog", pet_class).def(py::init<std::string>()); /* Another way of declaring a subclass relationship: reference parent's C++ type */ - py::class_<Rabbit, Pet>(m, "Rabbit") - .def(py::init<std::string>()); + py::class_<Rabbit, Pet>(m, "Rabbit").def(py::init<std::string>()); /* And another: list parent in class template arguments */ - py::class_<Hamster, Pet>(m, "Hamster") - .def(py::init<std::string>()); + py::class_<Hamster, Pet>(m, "Hamster").def(py::init<std::string>()); /* Constructors are not inherited by default */ py::class_<Chimera, Pet>(m, "Chimera"); - m.def("pet_name_species", [](const Pet &pet) { return pet.name() + " is a " + pet.species(); }); + m.def("pet_name_species", + [](const Pet &pet) { return pet.name() + " is a " + pet.species(); }); m.def("dog_bark", [](const Dog &dog) { return dog.bark(); }); // test_automatic_upcasting @@ -112,33 +150,35 @@ TEST_SUBMODULE(class_, m) { BaseClass(BaseClass &&) = default; virtual ~BaseClass() = default; }; - struct DerivedClass1 : BaseClass { }; - struct DerivedClass2 : BaseClass { }; + struct DerivedClass1 : BaseClass {}; + struct DerivedClass2 : BaseClass {}; py::class_<BaseClass>(m, "BaseClass").def(py::init<>()); py::class_<DerivedClass1>(m, "DerivedClass1").def(py::init<>()); py::class_<DerivedClass2>(m, "DerivedClass2").def(py::init<>()); - m.def("return_class_1", []() -> BaseClass* { return new DerivedClass1(); }); - m.def("return_class_2", []() -> BaseClass* { return new DerivedClass2(); }); - m.def("return_class_n", [](int n) -> BaseClass* { - if (n == 1) return new DerivedClass1(); - if (n == 2) return new DerivedClass2(); + m.def("return_class_1", []() -> BaseClass * { return new DerivedClass1(); }); + m.def("return_class_2", []() -> BaseClass * { return new DerivedClass2(); }); + m.def("return_class_n", [](int n) -> BaseClass * { + if (n == 1) { + return new DerivedClass1(); + } + if (n == 2) { + return new DerivedClass2(); + } return new BaseClass(); }); - m.def("return_none", []() -> BaseClass* { return nullptr; }); + m.def("return_none", []() -> BaseClass * { return nullptr; }); // test_isinstance - m.def("check_instances", [](py::list l) { - return py::make_tuple( - py::isinstance<py::tuple>(l[0]), - py::isinstance<py::dict>(l[1]), - py::isinstance<Pet>(l[2]), - py::isinstance<Pet>(l[3]), - py::isinstance<Dog>(l[4]), - py::isinstance<Rabbit>(l[5]), - py::isinstance<UnregisteredType>(l[6]) - ); + m.def("check_instances", [](const py::list &l) { + return py::make_tuple(py::isinstance<py::tuple>(l[0]), + py::isinstance<py::dict>(l[1]), + py::isinstance<Pet>(l[2]), + py::isinstance<Pet>(l[3]), + py::isinstance<Dog>(l[4]), + py::isinstance<Rabbit>(l[5]), + py::isinstance<UnregisteredType>(l[6])); }); struct Invalid {}; @@ -149,30 +189,24 @@ TEST_SUBMODULE(class_, m) { // See https://github.com/pybind/pybind11/issues/2486 // if (category == 2) // return py::type::of<int>(); - if (category == 1) + if (category == 1) { return py::type::of<DerivedClass1>(); - else - return py::type::of<Invalid>(); + } + return py::type::of<Invalid>(); }); - m.def("get_type_of", [](py::object ob) { - return py::type::of(ob); - }); + m.def("get_type_of", [](py::object ob) { return py::type::of(std::move(ob)); }); - m.def("get_type_classic", [](py::handle h) { - return h.get_type(); - }); + m.def("get_type_classic", [](py::handle h) { return h.get_type(); }); - m.def("as_type", [](py::object ob) { - return py::type(ob); - }); + m.def("as_type", [](const py::object &ob) { return py::type(ob); }); // test_mismatched_holder - struct MismatchBase1 { }; - struct MismatchDerived1 : MismatchBase1 { }; + struct MismatchBase1 {}; + struct MismatchDerived1 : MismatchBase1 {}; - struct MismatchBase2 { }; - struct MismatchDerived2 : MismatchBase2 { }; + struct MismatchBase2 {}; + struct MismatchDerived2 : MismatchBase2 {}; m.def("mismatched_holder_1", []() { auto mod = py::module_::import("__main__"); @@ -182,16 +216,14 @@ TEST_SUBMODULE(class_, m) { m.def("mismatched_holder_2", []() { auto mod = py::module_::import("__main__"); py::class_<MismatchBase2>(mod, "MismatchBase2"); - py::class_<MismatchDerived2, std::shared_ptr<MismatchDerived2>, - MismatchBase2>(mod, "MismatchDerived2"); + py::class_<MismatchDerived2, std::shared_ptr<MismatchDerived2>, MismatchBase2>( + mod, "MismatchDerived2"); }); // test_override_static // #511: problem with inheritance + overwritten def_static struct MyBase { - static std::unique_ptr<MyBase> make() { - return std::unique_ptr<MyBase>(new MyBase()); - } + static std::unique_ptr<MyBase> make() { return std::unique_ptr<MyBase>(new MyBase()); } }; struct MyDerived : MyBase { @@ -200,8 +232,7 @@ TEST_SUBMODULE(class_, m) { } }; - py::class_<MyBase>(m, "MyBase") - .def_static("make", &MyBase::make); + py::class_<MyBase>(m, "MyBase").def_static("make", &MyBase::make); py::class_<MyDerived, MyBase>(m, "MyDerived") .def_static("make", &MyDerived::make) @@ -211,15 +242,14 @@ TEST_SUBMODULE(class_, m) { struct ConvertibleFromUserType { int i; - ConvertibleFromUserType(UserType u) : i(u.value()) { } + explicit ConvertibleFromUserType(UserType u) : i(u.value()) {} }; - py::class_<ConvertibleFromUserType>(m, "AcceptsUserType") - .def(py::init<UserType>()); + py::class_<ConvertibleFromUserType>(m, "AcceptsUserType").def(py::init<UserType>()); py::implicitly_convertible<UserType, ConvertibleFromUserType>(); m.def("implicitly_convert_argument", [](const ConvertibleFromUserType &r) { return r.i; }); - m.def("implicitly_convert_variable", [](py::object o) { + m.def("implicitly_convert_variable", [](const py::object &o) { // `o` is `UserType` and `r` is a reference to a temporary created by implicit // conversion. This is valid when called inside a bound function because the temp // object is attached to the same life support system as the arguments. @@ -237,49 +267,91 @@ TEST_SUBMODULE(class_, m) { return py::str().release().ptr(); }; - auto def = new PyMethodDef{"f", f, METH_VARARGS, nullptr}; - py::capsule def_capsule(def, [](void *ptr) { delete reinterpret_cast<PyMethodDef *>(ptr); }); - return py::reinterpret_steal<py::object>(PyCFunction_NewEx(def, def_capsule.ptr(), m.ptr())); + auto *def = new PyMethodDef{"f", f, METH_VARARGS, nullptr}; + py::capsule def_capsule(def, + [](void *ptr) { delete reinterpret_cast<PyMethodDef *>(ptr); }); + return py::reinterpret_steal<py::object>( + PyCFunction_NewEx(def, def_capsule.ptr(), m.ptr())); }()); // test_operator_new_delete struct HasOpNewDel { std::uint64_t i; - static void *operator new(size_t s) { py::print("A new", s); return ::operator new(s); } - static void *operator new(size_t s, void *ptr) { py::print("A placement-new", s); return ptr; } - static void operator delete(void *p) { py::print("A delete"); return ::operator delete(p); } + static void *operator new(size_t s) { + py::print("A new", s); + return ::operator new(s); + } + static void *operator new(size_t s, void *ptr) { + py::print("A placement-new", s); + return ptr; + } + static void operator delete(void *p) { + py::print("A delete"); + return ::operator delete(p); + } }; struct HasOpNewDelSize { std::uint32_t i; - static void *operator new(size_t s) { py::print("B new", s); return ::operator new(s); } - static void *operator new(size_t s, void *ptr) { py::print("B placement-new", s); return ptr; } - static void operator delete(void *p, size_t s) { py::print("B delete", s); return ::operator delete(p); } + static void *operator new(size_t s) { + py::print("B new", s); + return ::operator new(s); + } + static void *operator new(size_t s, void *ptr) { + py::print("B placement-new", s); + return ptr; + } + static void operator delete(void *p, size_t s) { + py::print("B delete", s); + return ::operator delete(p); + } }; struct AliasedHasOpNewDelSize { std::uint64_t i; - static void *operator new(size_t s) { py::print("C new", s); return ::operator new(s); } - static void *operator new(size_t s, void *ptr) { py::print("C placement-new", s); return ptr; } - static void operator delete(void *p, size_t s) { py::print("C delete", s); return ::operator delete(p); } + static void *operator new(size_t s) { + py::print("C new", s); + return ::operator new(s); + } + static void *operator new(size_t s, void *ptr) { + py::print("C placement-new", s); + return ptr; + } + static void operator delete(void *p, size_t s) { + py::print("C delete", s); + return ::operator delete(p); + } virtual ~AliasedHasOpNewDelSize() = default; AliasedHasOpNewDelSize() = default; - AliasedHasOpNewDelSize(const AliasedHasOpNewDelSize&) = delete; + AliasedHasOpNewDelSize(const AliasedHasOpNewDelSize &) = delete; }; struct PyAliasedHasOpNewDelSize : AliasedHasOpNewDelSize { PyAliasedHasOpNewDelSize() = default; - PyAliasedHasOpNewDelSize(int) { } + explicit PyAliasedHasOpNewDelSize(int) {} std::uint64_t j; }; struct HasOpNewDelBoth { std::uint32_t i[8]; - static void *operator new(size_t s) { py::print("D new", s); return ::operator new(s); } - static void *operator new(size_t s, void *ptr) { py::print("D placement-new", s); return ptr; } - static void operator delete(void *p) { py::print("D delete"); return ::operator delete(p); } - static void operator delete(void *p, size_t s) { py::print("D wrong delete", s); return ::operator delete(p); } + static void *operator new(size_t s) { + py::print("D new", s); + return ::operator new(s); + } + static void *operator new(size_t s, void *ptr) { + py::print("D placement-new", s); + return ptr; + } + static void operator delete(void *p) { + py::print("D delete"); + return ::operator delete(p); + } + static void operator delete(void *p, size_t s) { + py::print("D wrong delete", s); + return ::operator delete(p); + } }; py::class_<HasOpNewDel>(m, "HasOpNewDel").def(py::init<>()); py::class_<HasOpNewDelSize>(m, "HasOpNewDelSize").def(py::init<>()); py::class_<HasOpNewDelBoth>(m, "HasOpNewDelBoth").def(py::init<>()); - py::class_<AliasedHasOpNewDelSize, PyAliasedHasOpNewDelSize> aliased(m, "AliasedHasOpNewDelSize"); + py::class_<AliasedHasOpNewDelSize, PyAliasedHasOpNewDelSize> aliased(m, + "AliasedHasOpNewDelSize"); aliased.def(py::init<>()); aliased.attr("size_noalias") = py::int_(sizeof(AliasedHasOpNewDelSize)); aliased.attr("size_alias") = py::int_(sizeof(PyAliasedHasOpNewDelSize)); @@ -302,13 +374,7 @@ TEST_SUBMODULE(class_, m) { using ProtectedA::foo; }; - py::class_<ProtectedA>(m, "ProtectedA") - .def(py::init<>()) -#if !defined(_MSC_VER) || _MSC_VER >= 1910 - .def("foo", &PublicistA::foo); -#else - .def("foo", static_cast<int (ProtectedA::*)() const>(&PublicistA::foo)); -#endif + py::class_<ProtectedA>(m, "ProtectedA").def(py::init<>()).def("foo", &PublicistA::foo); class ProtectedB { public: @@ -318,6 +384,8 @@ TEST_SUBMODULE(class_, m) { protected: virtual int foo() const { return value; } + virtual void *void_foo() { return static_cast<void *>(&value); } + virtual void *get_self() { return static_cast<void *>(this); } private: int value = 42; @@ -326,6 +394,8 @@ TEST_SUBMODULE(class_, m) { class TrampolineB : public ProtectedB { public: int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); } + void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); } + void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); } }; class PublicistB : public ProtectedB { @@ -333,17 +403,25 @@ TEST_SUBMODULE(class_, m) { // [workaround(intel)] = default does not work here // Removing or defaulting this destructor results in linking errors with the Intel compiler // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) - ~PublicistB() override {}; // NOLINT(modernize-use-equals-default) + ~PublicistB() override{}; // NOLINT(modernize-use-equals-default) using ProtectedB::foo; + using ProtectedB::get_self; + using ProtectedB::void_foo; }; + m.def("read_foo", [](const void *original) { + const int *ptr = reinterpret_cast<const int *>(original); + return *ptr; + }); + + m.def("pointers_equal", + [](const void *original, const void *comparison) { return original == comparison; }); + py::class_<ProtectedB, TrampolineB>(m, "ProtectedB") .def(py::init<>()) -#if !defined(_MSC_VER) || _MSC_VER >= 1910 - .def("foo", &PublicistB::foo); -#else - .def("foo", static_cast<int (ProtectedB::*)() const>(&PublicistB::foo)); -#endif + .def("foo", &PublicistB::foo) + .def("void_foo", &PublicistB::void_foo) + .def("get_self", &PublicistB::get_self); // test_brace_initialization struct BraceInitialization { @@ -383,8 +461,8 @@ TEST_SUBMODULE(class_, m) { py::class_<Nested>(base, "Nested") .def(py::init<>()) .def("fn", [](Nested &, int, NestBase &, Nested &) {}) - .def("fa", [](Nested &, int, NestBase &, Nested &) {}, - "a"_a, "b"_a, "c"_a); + .def( + "fa", [](Nested &, int, NestBase &, Nested &) {}, "a"_a, "b"_a, "c"_a); base.def("g", [](NestBase &, Nested &) {}); base.def("h", []() { return NestBase(); }); @@ -394,21 +472,21 @@ TEST_SUBMODULE(class_, m) { // generate a useful error message struct NotRegistered {}; - struct StringWrapper { std::string str; }; + struct StringWrapper { + std::string str; + }; m.def("test_error_after_conversions", [](int) {}); m.def("test_error_after_conversions", - [](StringWrapper) -> NotRegistered { return {}; }); + [](const StringWrapper &) -> NotRegistered { return {}; }); py::class_<StringWrapper>(m, "StringWrapper").def(py::init<std::string>()); py::implicitly_convertible<std::string, StringWrapper>(); - #if defined(PYBIND11_CPP17) - struct alignas(1024) Aligned { - std::uintptr_t ptr() const { return (uintptr_t) this; } - }; - py::class_<Aligned>(m, "Aligned") - .def(py::init<>()) - .def("ptr", &Aligned::ptr); - #endif +#if defined(PYBIND11_CPP17) + struct alignas(1024) Aligned { + std::uintptr_t ptr() const { return (uintptr_t) this; } + }; + py::class_<Aligned>(m, "Aligned").def(py::init<>()).def("ptr", &Aligned::ptr); +#endif // test_final struct IsFinal final {}; @@ -421,9 +499,7 @@ TEST_SUBMODULE(class_, m) { // test_exception_rvalue_abort struct PyPrintDestructor { PyPrintDestructor() = default; - ~PyPrintDestructor() { - py::print("Print from destructor"); - } + ~PyPrintDestructor() { py::print("Print from destructor"); } void throw_something() { throw std::runtime_error("error"); } }; py::class_<PyPrintDestructor>(m, "PyPrintDestructor") @@ -434,12 +510,10 @@ TEST_SUBMODULE(class_, m) { struct SamePointer {}; static SamePointer samePointer; py::class_<SamePointer, std::unique_ptr<SamePointer, py::nodelete>>(m, "SamePointer") - .def(py::init([]() { return &samePointer; })) - .def("__del__", [](SamePointer&) { py::print("__del__ called"); }); + .def(py::init([]() { return &samePointer; })); struct Empty {}; - py::class_<Empty>(m, "Empty") - .def(py::init<>()); + py::class_<Empty>(m, "Empty").def(py::init<>()); // test_base_and_derived_nested_scope struct BaseWithNested { @@ -462,30 +536,36 @@ TEST_SUBMODULE(class_, m) { struct OtherDuplicate {}; struct DuplicateNested {}; struct OtherDuplicateNested {}; - m.def("register_duplicate_class_name", [](py::module_ m) { + + m.def("register_duplicate_class_name", [](const py::module_ &m) { py::class_<Duplicate>(m, "Duplicate"); py::class_<OtherDuplicate>(m, "Duplicate"); }); - m.def("register_duplicate_class_type", [](py::module_ m) { + m.def("register_duplicate_class_type", [](const py::module_ &m) { py::class_<OtherDuplicate>(m, "OtherDuplicate"); py::class_<OtherDuplicate>(m, "YetAnotherDuplicate"); }); - m.def("register_duplicate_nested_class_name", [](py::object gt) { + m.def("register_duplicate_nested_class_name", [](const py::object >) { py::class_<DuplicateNested>(gt, "DuplicateNested"); py::class_<OtherDuplicateNested>(gt, "DuplicateNested"); }); - m.def("register_duplicate_nested_class_type", [](py::object gt) { + m.def("register_duplicate_nested_class_type", [](const py::object >) { py::class_<OtherDuplicateNested>(gt, "OtherDuplicateNested"); py::class_<OtherDuplicateNested>(gt, "YetAnotherDuplicateNested"); }); + + test_class::pr4220_tripped_over_this::bind_empty0(m); } -template <int N> class BreaksBase { public: +template <int N> +class BreaksBase { +public: virtual ~BreaksBase() = default; BreaksBase() = default; - BreaksBase(const BreaksBase&) = delete; + BreaksBase(const BreaksBase &) = delete; }; -template <int N> class BreaksTramp : public BreaksBase<N> {}; +template <int N> +class BreaksTramp : public BreaksBase<N> {}; // These should all compile just fine: using DoesntBreak1 = py::class_<BreaksBase<1>, std::unique_ptr<BreaksBase<1>>, BreaksTramp<1>>; using DoesntBreak2 = py::class_<BreaksBase<2>, BreaksTramp<2>, std::unique_ptr<BreaksBase<2>>>; @@ -495,43 +575,83 @@ using DoesntBreak5 = py::class_<BreaksBase<5>>; using DoesntBreak6 = py::class_<BreaksBase<6>, std::shared_ptr<BreaksBase<6>>, BreaksTramp<6>>; using DoesntBreak7 = py::class_<BreaksBase<7>, BreaksTramp<7>, std::shared_ptr<BreaksBase<7>>>; using DoesntBreak8 = py::class_<BreaksBase<8>, std::shared_ptr<BreaksBase<8>>>; -#define CHECK_BASE(N) static_assert(std::is_same<typename DoesntBreak##N::type, BreaksBase<N>>::value, \ - "DoesntBreak" #N " has wrong type!") -CHECK_BASE(1); CHECK_BASE(2); CHECK_BASE(3); CHECK_BASE(4); CHECK_BASE(5); CHECK_BASE(6); CHECK_BASE(7); CHECK_BASE(8); -#define CHECK_ALIAS(N) static_assert(DoesntBreak##N::has_alias && std::is_same<typename DoesntBreak##N::type_alias, BreaksTramp<N>>::value, \ +#define CHECK_BASE(N) \ + static_assert(std::is_same<typename DoesntBreak##N::type, BreaksBase<(N)>>::value, \ + "DoesntBreak" #N " has wrong type!") +CHECK_BASE(1); +CHECK_BASE(2); +CHECK_BASE(3); +CHECK_BASE(4); +CHECK_BASE(5); +CHECK_BASE(6); +CHECK_BASE(7); +CHECK_BASE(8); +#define CHECK_ALIAS(N) \ + static_assert( \ + DoesntBreak##N::has_alias \ + && std::is_same<typename DoesntBreak##N::type_alias, BreaksTramp<(N)>>::value, \ "DoesntBreak" #N " has wrong type_alias!") -#define CHECK_NOALIAS(N) static_assert(!DoesntBreak##N::has_alias && std::is_void<typename DoesntBreak##N::type_alias>::value, \ - "DoesntBreak" #N " has type alias, but shouldn't!") -CHECK_ALIAS(1); CHECK_ALIAS(2); CHECK_NOALIAS(3); CHECK_ALIAS(4); CHECK_NOALIAS(5); CHECK_ALIAS(6); CHECK_ALIAS(7); CHECK_NOALIAS(8); -#define CHECK_HOLDER(N, TYPE) static_assert(std::is_same<typename DoesntBreak##N::holder_type, std::TYPE##_ptr<BreaksBase<N>>>::value, \ - "DoesntBreak" #N " has wrong holder_type!") -CHECK_HOLDER(1, unique); CHECK_HOLDER(2, unique); CHECK_HOLDER(3, unique); CHECK_HOLDER(4, unique); CHECK_HOLDER(5, unique); -CHECK_HOLDER(6, shared); CHECK_HOLDER(7, shared); CHECK_HOLDER(8, shared); +#define CHECK_NOALIAS(N) \ + static_assert(!DoesntBreak##N::has_alias \ + && std::is_void<typename DoesntBreak##N::type_alias>::value, \ + "DoesntBreak" #N " has type alias, but shouldn't!") +CHECK_ALIAS(1); +CHECK_ALIAS(2); +CHECK_NOALIAS(3); +CHECK_ALIAS(4); +CHECK_NOALIAS(5); +CHECK_ALIAS(6); +CHECK_ALIAS(7); +CHECK_NOALIAS(8); +#define CHECK_HOLDER(N, TYPE) \ + static_assert(std::is_same<typename DoesntBreak##N::holder_type, \ + std::TYPE##_ptr<BreaksBase<(N)>>>::value, \ + "DoesntBreak" #N " has wrong holder_type!") +CHECK_HOLDER(1, unique); +CHECK_HOLDER(2, unique); +CHECK_HOLDER(3, unique); +CHECK_HOLDER(4, unique); +CHECK_HOLDER(5, unique); +CHECK_HOLDER(6, shared); +CHECK_HOLDER(7, shared); +CHECK_HOLDER(8, shared); // There's no nice way to test that these fail because they fail to compile; leave them here, // though, so that they can be manually tested by uncommenting them (and seeing that compilation // failures occurs). // We have to actually look into the type: the typedef alone isn't enough to instantiate the type: -#define CHECK_BROKEN(N) static_assert(std::is_same<typename Breaks##N::type, BreaksBase<-N>>::value, \ - "Breaks1 has wrong type!"); - -//// Two holder classes: -//typedef py::class_<BreaksBase<-1>, std::unique_ptr<BreaksBase<-1>>, std::unique_ptr<BreaksBase<-1>>> Breaks1; -//CHECK_BROKEN(1); -//// Two aliases: -//typedef py::class_<BreaksBase<-2>, BreaksTramp<-2>, BreaksTramp<-2>> Breaks2; -//CHECK_BROKEN(2); -//// Holder + 2 aliases -//typedef py::class_<BreaksBase<-3>, std::unique_ptr<BreaksBase<-3>>, BreaksTramp<-3>, BreaksTramp<-3>> Breaks3; -//CHECK_BROKEN(3); -//// Alias + 2 holders -//typedef py::class_<BreaksBase<-4>, std::unique_ptr<BreaksBase<-4>>, BreaksTramp<-4>, std::shared_ptr<BreaksBase<-4>>> Breaks4; -//CHECK_BROKEN(4); -//// Invalid option (not a subclass or holder) -//typedef py::class_<BreaksBase<-5>, BreaksTramp<-4>> Breaks5; -//CHECK_BROKEN(5); -//// Invalid option: multiple inheritance not supported: -//template <> struct BreaksBase<-8> : BreaksBase<-6>, BreaksBase<-7> {}; -//typedef py::class_<BreaksBase<-8>, BreaksBase<-6>, BreaksBase<-7>> Breaks8; -//CHECK_BROKEN(8); +#define CHECK_BROKEN(N) \ + static_assert(std::is_same<typename Breaks##N::type, BreaksBase<-(N)>>::value, \ + "Breaks1 has wrong type!"); + +#ifdef PYBIND11_NEVER_DEFINED_EVER +// Two holder classes: +typedef py:: + class_<BreaksBase<-1>, std::unique_ptr<BreaksBase<-1>>, std::unique_ptr<BreaksBase<-1>>> + Breaks1; +CHECK_BROKEN(1); +// Two aliases: +typedef py::class_<BreaksBase<-2>, BreaksTramp<-2>, BreaksTramp<-2>> Breaks2; +CHECK_BROKEN(2); +// Holder + 2 aliases +typedef py:: + class_<BreaksBase<-3>, std::unique_ptr<BreaksBase<-3>>, BreaksTramp<-3>, BreaksTramp<-3>> + Breaks3; +CHECK_BROKEN(3); +// Alias + 2 holders +typedef py::class_<BreaksBase<-4>, + std::unique_ptr<BreaksBase<-4>>, + BreaksTramp<-4>, + std::shared_ptr<BreaksBase<-4>>> + Breaks4; +CHECK_BROKEN(4); +// Invalid option (not a subclass or holder) +typedef py::class_<BreaksBase<-5>, BreaksTramp<-4>> Breaks5; +CHECK_BROKEN(5); +// Invalid option: multiple inheritance not supported: +template <> +struct BreaksBase<-8> : BreaksBase<-6>, BreaksBase<-7> {}; +typedef py::class_<BreaksBase<-8>, BreaksBase<-6>, BreaksBase<-7>> Breaks8; +CHECK_BROKEN(8); +#endif diff --git a/tests/test_class.py b/tests/test_class.py index bdcced96..ee7467cf 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -1,14 +1,17 @@ -# -*- coding: utf-8 -*- import pytest -import env # noqa: F401 - +import env +from pybind11_tests import ConstructorStats, UserType from pybind11_tests import class_ as m -from pybind11_tests import UserType, ConstructorStats + + +def test_obj_class_name(): + expected_name = "UserType" if env.PYPY else "pybind11_tests.UserType" + assert m.obj_class_name(UserType(1)) == expected_name + assert m.obj_class_name(UserType) == expected_name def test_repr(): - # In Python 3.3+, repr() accesses __qualname__ assert "pybind11_type" in repr(type(UserType)) assert "UserType" in repr(UserType) @@ -26,6 +29,14 @@ def test_instance(msg): assert cstats.alive() == 0 +def test_instance_new(): + instance = m.NoConstructorNew() # .__new__(m.NoConstructor.__class__) + cstats = ConstructorStats.get(m.NoConstructorNew) + assert cstats.alive() == 1 + del instance + assert cstats.alive() == 0 + + def test_type(): assert m.check_type(1) == m.DerivedClass1 with pytest.raises(RuntimeError) as execinfo: @@ -96,8 +107,8 @@ def test_docstrings(doc): def test_qualname(doc): - """Tests that a properly qualified name is set in __qualname__ (even in pre-3.3, where we - backport the attribute) and that generated docstrings properly use it and the module name""" + """Tests that a properly qualified name is set in __qualname__ and that + generated docstrings properly use it and the module name""" assert m.NestBase.__qualname__ == "NestBase" assert m.NestBase.Nested.__qualname__ == "NestBase.Nested" @@ -123,13 +134,13 @@ def test_qualname(doc): doc(m.NestBase.Nested.fn) == """ fn(self: m.class_.NestBase.Nested, arg0: int, arg1: m.class_.NestBase, arg2: m.class_.NestBase.Nested) -> None - """ # noqa: E501 line too long + """ ) assert ( doc(m.NestBase.Nested.fa) == """ fa(self: m.class_.NestBase.Nested, a: int, b: m.class_.NestBase, c: m.class_.NestBase.Nested) -> None - """ # noqa: E501 line too long + """ ) assert m.NestBase.__module__ == "pybind11_tests.class_" assert m.NestBase.Nested.__module__ == "pybind11_tests.class_" @@ -171,7 +182,6 @@ def test_inheritance(msg): def test_inheritance_init(msg): - # Single base class Python(m.Pet): def __init__(self): @@ -208,7 +218,7 @@ def test_automatic_upcasting(): def test_isinstance(): - objects = [tuple(), dict(), m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 + objects = [(), {}, m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 expected = (True, True, True, True, True, False, False) assert m.check_instances(objects) == expected @@ -308,6 +318,8 @@ def test_bind_protected_functions(): b = m.ProtectedB() assert b.foo() == 42 + assert m.read_foo(b.void_foo()) == 42 + assert m.pointers_equal(b.get_self(), b) class C(m.ProtectedB): def __init__(self): @@ -321,7 +333,7 @@ def test_bind_protected_functions(): def test_brace_initialization(): - """ Tests that simple POD classes can be constructed using C++11 brace initialization """ + """Tests that simple POD classes can be constructed using C++11 brace initialization""" a = m.BraceInitialization(123, "test") assert a.field1 == 123 assert a.field2 == "test" @@ -412,7 +424,7 @@ def test_exception_rvalue_abort(): # https://github.com/pybind/pybind11/issues/1568 -def test_multiple_instances_with_same_pointer(capture): +def test_multiple_instances_with_same_pointer(): n = 100 instances = [m.SamePointer() for _ in range(n)] for i in range(n): @@ -464,3 +476,10 @@ def test_register_duplicate_class(): m.register_duplicate_nested_class_type(ClassScope) expected = 'generic_type: type "YetAnotherDuplicateNested" is already registered!' assert str(exc_info.value) == expected + + +def test_pr4220_tripped_over_this(): + assert ( + m.Empty0().get_msg() + == "This is really only meant to exercise successful compilation." + ) diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 8bfaa386..e5aa975c 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -1,6 +1,3 @@ -# Built-in in CMake 3.5+ -include(CMakeParseArguments) - add_custom_target(test_cmake_build) function(pybind11_add_build_test name) diff --git a/tests/test_cmake_build/embed.cpp b/tests/test_cmake_build/embed.cpp index a3abc8a8..30bc4f1e 100644 --- a/tests/test_cmake_build/embed.cpp +++ b/tests/test_cmake_build/embed.cpp @@ -6,15 +6,17 @@ PYBIND11_EMBEDDED_MODULE(test_cmake_build, m) { } int main(int argc, char *argv[]) { - if (argc != 2) + if (argc != 2) { throw std::runtime_error("Expected test.py file as the first argument"); - auto test_py_file = argv[1]; + } + auto *test_py_file = argv[1]; py::scoped_interpreter guard{}; auto m = py::module_::import("test_cmake_build"); - if (m.attr("add")(1, 2).cast<int>() != 3) + if (m.attr("add")(1, 2).cast<int>() != 3) { throw std::runtime_error("embed.cpp failed"); + } py::module_::import("sys").attr("argv") = py::make_tuple("test.py", "embed.cpp"); py::eval_file(test_py_file, py::globals()); diff --git a/tests/test_cmake_build/installed_embed/CMakeLists.txt b/tests/test_cmake_build/installed_embed/CMakeLists.txt index 64ae5c4b..d9dcb45e 100644 --- a/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_embed CXX) @@ -22,5 +22,7 @@ set_target_properties(test_installed_embed PROPERTIES OUTPUT_NAME test_cmake_bui # This may be needed to resolve header conflicts, e.g. between Python release and debug headers. set_target_properties(test_installed_embed PROPERTIES NO_SYSTEM_FROM_IMPORTED ON) -add_custom_target(check_installed_embed $<TARGET_FILE:test_installed_embed> - ${PROJECT_SOURCE_DIR}/../test.py) +add_custom_target( + check_installed_embed + $<TARGET_FILE:test_installed_embed> ${PROJECT_SOURCE_DIR}/../test.py + DEPENDS test_installed_embed) diff --git a/tests/test_cmake_build/installed_function/CMakeLists.txt b/tests/test_cmake_build/installed_function/CMakeLists.txt index 1a502863..2f4f6427 100644 --- a/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,13 +1,13 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) project(test_installed_module CXX) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_function CXX) @@ -35,4 +35,5 @@ add_custom_target( PYTHONPATH=$<TARGET_FILE_DIR:test_installed_function> ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py - ${PROJECT_NAME}) + ${PROJECT_NAME} + DEPENDS test_installed_function) diff --git a/tests/test_cmake_build/installed_target/CMakeLists.txt b/tests/test_cmake_build/installed_target/CMakeLists.txt index b38eb774..a981e236 100644 --- a/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_target CXX) @@ -42,4 +42,5 @@ add_custom_target( PYTHONPATH=$<TARGET_FILE_DIR:test_installed_target> ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py - ${PROJECT_NAME}) + ${PROJECT_NAME} + DEPENDS test_installed_target) diff --git a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index dfb9cb8f..f286746b 100644 --- a/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_embed CXX) @@ -23,8 +23,10 @@ add_executable(test_subdirectory_embed ../embed.cpp) target_link_libraries(test_subdirectory_embed PRIVATE pybind11::embed) set_target_properties(test_subdirectory_embed PROPERTIES OUTPUT_NAME test_cmake_build) -add_custom_target(check_subdirectory_embed $<TARGET_FILE:test_subdirectory_embed> - "${PROJECT_SOURCE_DIR}/../test.py") +add_custom_target( + check_subdirectory_embed + $<TARGET_FILE:test_subdirectory_embed> "${PROJECT_SOURCE_DIR}/../test.py" + DEPENDS test_subdirectory_embed) # Test custom export group -- PYBIND11_EXPORT_NAME add_library(test_embed_lib ../embed.cpp) diff --git a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index 34aedcf6..275a75c0 100644 --- a/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_function CXX) @@ -31,4 +31,5 @@ add_custom_target( PYTHONPATH=$<TARGET_FILE_DIR:test_subdirectory_function> ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py - ${PROJECT_NAME}) + ${PROJECT_NAME} + DEPENDS test_subdirectory_function) diff --git a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index 31d862f6..37bb2c56 100644 --- a/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_target CXX) @@ -37,4 +37,5 @@ add_custom_target( PYTHONPATH=$<TARGET_FILE_DIR:test_subdirectory_target> ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py - ${PROJECT_NAME}) + ${PROJECT_NAME} + DEPENDS test_subdirectory_target) diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index 87ed5135..807fd43b 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -1,6 +1,8 @@ -# -*- coding: utf-8 -*- import sys + import test_cmake_build +assert isinstance(__file__, str) # Test this is properly set + assert test_cmake_build.add(1, 2) == 3 -print("{} imports, runs, and adds: 1 + 2 = 3".format(sys.argv[1])) +print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") diff --git a/tests/test_const_name.cpp b/tests/test_const_name.cpp new file mode 100644 index 00000000..2ad01e68 --- /dev/null +++ b/tests/test_const_name.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2021 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11_tests.h" + +// IUT = Implementation Under Test +#define CONST_NAME_TESTS(TEST_FUNC, IUT) \ + std::string TEST_FUNC(int selector) { \ + switch (selector) { \ + case 0: \ + return IUT("").text; \ + case 1: \ + return IUT("A").text; \ + case 2: \ + return IUT("Bd").text; \ + case 3: \ + return IUT("Cef").text; \ + case 4: \ + return IUT<int>().text; /*NOLINT(bugprone-macro-parentheses)*/ \ + case 5: \ + return IUT<std::string>().text; /*NOLINT(bugprone-macro-parentheses)*/ \ + case 6: \ + return IUT<true>("T1", "T2").text; /*NOLINT(bugprone-macro-parentheses)*/ \ + case 7: \ + return IUT<false>("U1", "U2").text; /*NOLINT(bugprone-macro-parentheses)*/ \ + case 8: \ + /*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \ + return IUT<true>(IUT("D1"), IUT("D2")).text; \ + case 9: \ + /*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \ + return IUT<false>(IUT("E1"), IUT("E2")).text; \ + case 10: \ + return IUT("KeepAtEnd").text; \ + default: \ + break; \ + } \ + throw std::runtime_error("Invalid selector value."); \ + } + +CONST_NAME_TESTS(const_name_tests, py::detail::const_name) + +#ifdef PYBIND11_DETAIL_UNDERSCORE_BACKWARD_COMPATIBILITY +CONST_NAME_TESTS(underscore_tests, py::detail::_) +#endif + +TEST_SUBMODULE(const_name, m) { + m.def("const_name_tests", const_name_tests); + +#if defined(PYBIND11_DETAIL_UNDERSCORE_BACKWARD_COMPATIBILITY) + m.def("underscore_tests", underscore_tests); +#else + m.attr("underscore_tests") = "PYBIND11_DETAIL_UNDERSCORE_BACKWARD_COMPATIBILITY not defined."; +#endif +} diff --git a/tests/test_const_name.py b/tests/test_const_name.py new file mode 100644 index 00000000..a145f0bb --- /dev/null +++ b/tests/test_const_name.py @@ -0,0 +1,29 @@ +import pytest + +from pybind11_tests import const_name as m + + +@pytest.mark.parametrize("func", [m.const_name_tests, m.underscore_tests]) +@pytest.mark.parametrize( + ("selector", "expected"), + enumerate( + ( + "", + "A", + "Bd", + "Cef", + "%", + "%", + "T1", + "U2", + "D1", + "E2", + "KeepAtEnd", + ) + ), +) +def test_const_name(func, selector, expected): + if isinstance(func, str): + pytest.skip(func) + text = func(selector) + assert text == expected diff --git a/tests/test_constants_and_functions.cpp b/tests/test_constants_and_functions.cpp index 8855dd7d..312edca9 100644 --- a/tests/test_constants_and_functions.cpp +++ b/tests/test_constants_and_functions.cpp @@ -1,5 +1,6 @@ /* - tests/test_constants_and_functions.cpp -- global constants and functions, enumerations, raw byte strings + tests/test_constants_and_functions.cpp -- global constants and functions, enumerations, raw + byte strings Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch> @@ -11,20 +12,14 @@ enum MyEnum { EFirstEntry = 1, ESecondEntry }; -std::string test_function1() { - return "test_function()"; -} +std::string test_function1() { return "test_function()"; } -std::string test_function2(MyEnum k) { - return "test_function(enum=" + std::to_string(k) + ")"; -} +std::string test_function2(MyEnum k) { return "test_function(enum=" + std::to_string(k) + ")"; } -std::string test_function3(int i) { - return "test_function(" + std::to_string(i) + ")"; -} +std::string test_function3(int i) { return "test_function(" + std::to_string(i) + ")"; } -py::str test_function4() { return "test_function()"; } -py::str test_function4(char *) { return "test_function(char *)"; } +py::str test_function4() { return "test_function()"; } +py::str test_function4(char *) { return "test_function(char *)"; } py::str test_function4(int, float) { return "test_function(int, float)"; } py::str test_function4(float, int) { return "test_function(float, int)"; } @@ -33,57 +28,54 @@ py::bytes return_bytes() { return std::string(data, 4); } -std::string print_bytes(py::bytes bytes) { +std::string print_bytes(const py::bytes &bytes) { std::string ret = "bytes["; const auto value = static_cast<std::string>(bytes); - for (size_t i = 0; i < value.length(); ++i) { - ret += std::to_string(static_cast<int>(value[i])) + " "; + for (char c : value) { + ret += std::to_string(static_cast<int>(c)) + ' '; } ret.back() = ']'; return ret; } -// Test that we properly handle C++17 exception specifiers (which are part of the function signature -// in C++17). These should all still work before C++17, but don't affect the function signature. +// Test that we properly handle C++17 exception specifiers (which are part of the function +// signature in C++17). These should all still work before C++17, but don't affect the function +// signature. namespace test_exc_sp { // [workaround(intel)] Unable to use noexcept instead of noexcept(true) // Make the f1 test basically the same as the f2 test in C++17 mode for the Intel compiler as // it fails to compile with a plain noexcept (tested with icc (ICC) 2021.1 Beta 20200827). #if defined(__INTEL_COMPILER) && defined(PYBIND11_CPP17) -int f1(int x) noexcept(true) { return x+1; } +int f1(int x) noexcept(true) { return x + 1; } #else -int f1(int x) noexcept { return x+1; } -#endif -int f2(int x) noexcept(true) { return x+2; } -int f3(int x) noexcept(false) { return x+3; } -#if defined(__GNUG__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif -int f4(int x) throw() { return x+4; } // Deprecated equivalent to noexcept(true) -#if defined(__GNUG__) -# pragma GCC diagnostic pop +int f1(int x) noexcept { return x + 1; } #endif +int f2(int x) noexcept(true) { return x + 2; } +int f3(int x) noexcept(false) { return x + 3; } +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") +PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") +// NOLINTNEXTLINE(modernize-use-noexcept) +int f4(int x) throw() { return x + 4; } // Deprecated equivalent to noexcept(true) +PYBIND11_WARNING_POP struct C { - int m1(int x) noexcept { return x-1; } - int m2(int x) const noexcept { return x-2; } - int m3(int x) noexcept(true) { return x-3; } - int m4(int x) const noexcept(true) { return x-4; } - int m5(int x) noexcept(false) { return x-5; } - int m6(int x) const noexcept(false) { return x-6; } -#if defined(__GNUG__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif - int m7(int x) throw() { return x-7; } - int m8(int x) const throw() { return x-8; } -#if defined(__GNUG__) -# pragma GCC diagnostic pop -#endif + int m1(int x) noexcept { return x - 1; } + int m2(int x) const noexcept { return x - 2; } + int m3(int x) noexcept(true) { return x - 3; } + int m4(int x) const noexcept(true) { return x - 4; } + int m5(int x) noexcept(false) { return x - 5; } + int m6(int x) const noexcept(false) { return x - 6; } + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") + PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") + // NOLINTNEXTLINE(modernize-use-noexcept) + int m7(int x) throw() { return x - 7; } + // NOLINTNEXTLINE(modernize-use-noexcept) + int m8(int x) const throw() { return x - 8; } + PYBIND11_WARNING_POP }; } // namespace test_exc_sp - TEST_SUBMODULE(constants_and_functions, m) { // test_constants m.attr("some_constant") = py::int_(14); @@ -125,25 +117,38 @@ TEST_SUBMODULE(constants_and_functions, m) { .def("m5", &C::m5) .def("m6", &C::m6) .def("m7", &C::m7) - .def("m8", &C::m8) - ; + .def("m8", &C::m8); m.def("f1", f1); m.def("f2", f2); + + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_INTEL(878) // incompatible exception specifications m.def("f3", f3); + PYBIND11_WARNING_POP + m.def("f4", f4); // test_function_record_leaks - struct LargeCapture { + m.def("register_large_capture_with_invalid_arguments", [](py::module_ m) { // This should always be enough to trigger the alternative branch // where `sizeof(capture) > sizeof(rec->data)` - uint64_t zeros[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - }; - m.def("register_large_capture_with_invalid_arguments", [](py::module_ m) { - LargeCapture capture; // VS 2015's MSVC is acting up if we create the array here - m.def("should_raise", [capture](int) { return capture.zeros[9] + 33; }, py::kw_only(), py::arg()); + uint64_t capture[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; +#if defined(__GNUC__) && __GNUC__ == 4 // CentOS7 + py::detail::silence_unused_warnings(capture); +#endif + m.def( + "should_raise", [capture](int) { return capture[9] + 33; }, py::kw_only(), py::arg()); }); - m.def("register_with_raising_repr", [](py::module_ m, py::object default_value) { - m.def("should_raise", [](int, int, py::object) { return 42; }, "some docstring", - py::arg_v("x", 42), py::arg_v("y", 42, "<the answer>"), py::arg_v("z", default_value)); + m.def("register_with_raising_repr", [](py::module_ m, const py::object &default_value) { + m.def( + "should_raise", + [](int, int, const py::object &) { return 42; }, + "some docstring", + py::arg_v("x", 42), + py::arg_v("y", 42, "<the answer>"), + py::arg_v("z", default_value)); }); + + // test noexcept(true) lambda (#4565) + m.def("l1", []() noexcept(true) { return 0; }); } diff --git a/tests/test_constants_and_functions.py b/tests/test_constants_and_functions.py index ff13bd0f..a1142461 100644 --- a/tests/test_constants_and_functions.py +++ b/tests/test_constants_and_functions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import pytest m = pytest.importorskip("pybind11_tests.constants_and_functions") @@ -51,3 +50,7 @@ def test_function_record_leaks(): m.register_large_capture_with_invalid_arguments(m) with pytest.raises(RuntimeError): m.register_with_raising_repr(m, RaisingRepr()) + + +def test_noexcept_lambda(): + assert m.l1() == 0 diff --git a/tests/test_copy_move.cpp b/tests/test_copy_move.cpp index 322e9bb8..f5473355 100644 --- a/tests/test_copy_move.cpp +++ b/tests/test_copy_move.cpp @@ -8,38 +8,50 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/stl.h> +#include "constructor_stats.h" +#include "pybind11_tests.h" + +#include <type_traits> + template <typename derived> struct empty { - static const derived& get_one() { return instance_; } + static const derived &get_one() { return instance_; } static derived instance_; }; struct lacking_copy_ctor : public empty<lacking_copy_ctor> { lacking_copy_ctor() = default; - lacking_copy_ctor(const lacking_copy_ctor& other) = delete; + lacking_copy_ctor(const lacking_copy_ctor &other) = delete; }; -template <> lacking_copy_ctor empty<lacking_copy_ctor>::instance_ = {}; +template <> +lacking_copy_ctor empty<lacking_copy_ctor>::instance_ = {}; struct lacking_move_ctor : public empty<lacking_move_ctor> { lacking_move_ctor() = default; - lacking_move_ctor(const lacking_move_ctor& other) = delete; - lacking_move_ctor(lacking_move_ctor&& other) = delete; + lacking_move_ctor(const lacking_move_ctor &other) = delete; + lacking_move_ctor(lacking_move_ctor &&other) = delete; }; -template <> lacking_move_ctor empty<lacking_move_ctor>::instance_ = {}; +template <> +lacking_move_ctor empty<lacking_move_ctor>::instance_ = {}; /* Custom type caster move/copy test classes */ class MoveOnlyInt { public: MoveOnlyInt() { print_default_created(this); } - MoveOnlyInt(int v) : value{std::move(v)} { print_created(this, value); } - MoveOnlyInt(MoveOnlyInt &&m) { print_move_created(this, m.value); std::swap(value, m.value); } - MoveOnlyInt &operator=(MoveOnlyInt &&m) { print_move_assigned(this, m.value); std::swap(value, m.value); return *this; } + explicit MoveOnlyInt(int v) : value{v} { print_created(this, value); } + MoveOnlyInt(MoveOnlyInt &&m) noexcept { + print_move_created(this, m.value); + std::swap(value, m.value); + } + MoveOnlyInt &operator=(MoveOnlyInt &&m) noexcept { + print_move_assigned(this, m.value); + std::swap(value, m.value); + return *this; + } MoveOnlyInt(const MoveOnlyInt &) = delete; MoveOnlyInt &operator=(const MoveOnlyInt &) = delete; ~MoveOnlyInt() { print_destroyed(this); } @@ -49,11 +61,26 @@ public: class MoveOrCopyInt { public: MoveOrCopyInt() { print_default_created(this); } - MoveOrCopyInt(int v) : value{std::move(v)} { print_created(this, value); } - MoveOrCopyInt(MoveOrCopyInt &&m) { print_move_created(this, m.value); std::swap(value, m.value); } - MoveOrCopyInt &operator=(MoveOrCopyInt &&m) { print_move_assigned(this, m.value); std::swap(value, m.value); return *this; } - MoveOrCopyInt(const MoveOrCopyInt &c) { print_copy_created(this, c.value); value = c.value; } - MoveOrCopyInt &operator=(const MoveOrCopyInt &c) { print_copy_assigned(this, c.value); value = c.value; return *this; } + explicit MoveOrCopyInt(int v) : value{v} { print_created(this, value); } + MoveOrCopyInt(MoveOrCopyInt &&m) noexcept { + print_move_created(this, m.value); + std::swap(value, m.value); + } + MoveOrCopyInt &operator=(MoveOrCopyInt &&m) noexcept { + print_move_assigned(this, m.value); + std::swap(value, m.value); + return *this; + } + MoveOrCopyInt(const MoveOrCopyInt &c) { + print_copy_created(this, c.value); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + value = c.value; + } + MoveOrCopyInt &operator=(const MoveOrCopyInt &c) { + print_copy_assigned(this, c.value); + value = c.value; + return *this; + } ~MoveOrCopyInt() { print_destroyed(this); } int value; @@ -61,41 +88,71 @@ public: class CopyOnlyInt { public: CopyOnlyInt() { print_default_created(this); } - CopyOnlyInt(int v) : value{std::move(v)} { print_created(this, value); } - CopyOnlyInt(const CopyOnlyInt &c) { print_copy_created(this, c.value); value = c.value; } - CopyOnlyInt &operator=(const CopyOnlyInt &c) { print_copy_assigned(this, c.value); value = c.value; return *this; } + explicit CopyOnlyInt(int v) : value{v} { print_created(this, value); } + CopyOnlyInt(const CopyOnlyInt &c) { + print_copy_created(this, c.value); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + value = c.value; + } + CopyOnlyInt &operator=(const CopyOnlyInt &c) { + print_copy_assigned(this, c.value); + value = c.value; + return *this; + } ~CopyOnlyInt() { print_destroyed(this); } int value; }; PYBIND11_NAMESPACE_BEGIN(pybind11) PYBIND11_NAMESPACE_BEGIN(detail) -template <> struct type_caster<MoveOnlyInt> { - PYBIND11_TYPE_CASTER(MoveOnlyInt, _("MoveOnlyInt")); - bool load(handle src, bool) { value = MoveOnlyInt(src.cast<int>()); return true; } - static handle cast(const MoveOnlyInt &m, return_value_policy r, handle p) { return pybind11::cast(m.value, r, p); } +template <> +struct type_caster<MoveOnlyInt> { + PYBIND11_TYPE_CASTER(MoveOnlyInt, const_name("MoveOnlyInt")); + bool load(handle src, bool) { + value = MoveOnlyInt(src.cast<int>()); + return true; + } + static handle cast(const MoveOnlyInt &m, return_value_policy r, handle p) { + return pybind11::cast(m.value, r, p); + } }; -template <> struct type_caster<MoveOrCopyInt> { - PYBIND11_TYPE_CASTER(MoveOrCopyInt, _("MoveOrCopyInt")); - bool load(handle src, bool) { value = MoveOrCopyInt(src.cast<int>()); return true; } - static handle cast(const MoveOrCopyInt &m, return_value_policy r, handle p) { return pybind11::cast(m.value, r, p); } +template <> +struct type_caster<MoveOrCopyInt> { + PYBIND11_TYPE_CASTER(MoveOrCopyInt, const_name("MoveOrCopyInt")); + bool load(handle src, bool) { + value = MoveOrCopyInt(src.cast<int>()); + return true; + } + static handle cast(const MoveOrCopyInt &m, return_value_policy r, handle p) { + return pybind11::cast(m.value, r, p); + } }; -template <> struct type_caster<CopyOnlyInt> { +template <> +struct type_caster<CopyOnlyInt> { protected: CopyOnlyInt value; + public: - static constexpr auto name = _("CopyOnlyInt"); - bool load(handle src, bool) { value = CopyOnlyInt(src.cast<int>()); return true; } - static handle cast(const CopyOnlyInt &m, return_value_policy r, handle p) { return pybind11::cast(m.value, r, p); } + static constexpr auto name = const_name("CopyOnlyInt"); + bool load(handle src, bool) { + value = CopyOnlyInt(src.cast<int>()); + return true; + } + static handle cast(const CopyOnlyInt &m, return_value_policy r, handle p) { + return pybind11::cast(m.value, r, p); + } static handle cast(const CopyOnlyInt *src, return_value_policy policy, handle parent) { - if (!src) return none().release(); + if (!src) { + return none().release(); + } return cast(*src, policy, parent); } - operator CopyOnlyInt*() { return &value; } - operator CopyOnlyInt&() { return value; } - template <typename T> using cast_op_type = pybind11::detail::cast_op_type<T>; + explicit operator CopyOnlyInt *() { return &value; } + explicit operator CopyOnlyInt &() { return value; } + template <typename T> + using cast_op_type = pybind11::detail::cast_op_type<T>; }; PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) @@ -103,22 +160,21 @@ PYBIND11_NAMESPACE_END(pybind11) TEST_SUBMODULE(copy_move_policies, m) { // test_lacking_copy_ctor py::class_<lacking_copy_ctor>(m, "lacking_copy_ctor") - .def_static("get_one", &lacking_copy_ctor::get_one, - py::return_value_policy::copy); + .def_static("get_one", &lacking_copy_ctor::get_one, py::return_value_policy::copy); // test_lacking_move_ctor py::class_<lacking_move_ctor>(m, "lacking_move_ctor") - .def_static("get_one", &lacking_move_ctor::get_one, - py::return_value_policy::move); + .def_static("get_one", &lacking_move_ctor::get_one, py::return_value_policy::move); // test_move_and_copy_casts - m.def("move_and_copy_casts", [](py::object o) { + // NOLINTNEXTLINE(performance-unnecessary-value-param) + m.def("move_and_copy_casts", [](const py::object &o) { int r = 0; r += py::cast<MoveOrCopyInt>(o).value; /* moves */ - r += py::cast<MoveOnlyInt>(o).value; /* moves */ - r += py::cast<CopyOnlyInt>(o).value; /* copies */ - auto m1(py::cast<MoveOrCopyInt>(o)); /* moves */ - auto m2(py::cast<MoveOnlyInt>(o)); /* moves */ - auto m3(py::cast<CopyOnlyInt>(o)); /* copies */ + r += py::cast<MoveOnlyInt>(o).value; /* moves */ + r += py::cast<CopyOnlyInt>(o).value; /* copies */ + auto m1(py::cast<MoveOrCopyInt>(o)); /* moves */ + auto m2(py::cast<MoveOnlyInt>(o)); /* moves */ + auto m3(py::cast<CopyOnlyInt>(o)); /* copies */ r += m1.value + m2.value + m3.value; return r; @@ -126,30 +182,40 @@ TEST_SUBMODULE(copy_move_policies, m) { // test_move_and_copy_loads m.def("move_only", [](MoveOnlyInt m) { return m.value; }); + // Changing this breaks the existing test: needs careful review. + // NOLINTNEXTLINE(performance-unnecessary-value-param) m.def("move_or_copy", [](MoveOrCopyInt m) { return m.value; }); + // Changing this breaks the existing test: needs careful review. + // NOLINTNEXTLINE(performance-unnecessary-value-param) m.def("copy_only", [](CopyOnlyInt m) { return m.value; }); - m.def("move_pair", [](std::pair<MoveOnlyInt, MoveOrCopyInt> p) { - return p.first.value + p.second.value; - }); + m.def("move_pair", + [](std::pair<MoveOnlyInt, MoveOrCopyInt> p) { return p.first.value + p.second.value; }); m.def("move_tuple", [](std::tuple<MoveOnlyInt, MoveOrCopyInt, MoveOnlyInt> t) { return std::get<0>(t).value + std::get<1>(t).value + std::get<2>(t).value; }); m.def("copy_tuple", [](std::tuple<CopyOnlyInt, CopyOnlyInt> t) { return std::get<0>(t).value + std::get<1>(t).value; }); - m.def("move_copy_nested", [](std::pair<MoveOnlyInt, std::pair<std::tuple<MoveOrCopyInt, CopyOnlyInt, std::tuple<MoveOnlyInt>>, MoveOrCopyInt>> x) { - return x.first.value + std::get<0>(x.second.first).value + std::get<1>(x.second.first).value + - std::get<0>(std::get<2>(x.second.first)).value + x.second.second.value; - }); + m.def("move_copy_nested", + [](std::pair<MoveOnlyInt, + std::pair<std::tuple<MoveOrCopyInt, CopyOnlyInt, std::tuple<MoveOnlyInt>>, + MoveOrCopyInt>> x) { + return x.first.value + std::get<0>(x.second.first).value + + std::get<1>(x.second.first).value + + std::get<0>(std::get<2>(x.second.first)).value + x.second.second.value; + }); m.def("move_and_copy_cstats", []() { ConstructorStats::gc(); // Reset counts to 0 so that previous tests don't affect later ones: auto &mc = ConstructorStats::get<MoveOrCopyInt>(); - mc.move_assignments = mc.move_constructions = mc.copy_assignments = mc.copy_constructions = 0; + mc.move_assignments = mc.move_constructions = mc.copy_assignments = mc.copy_constructions + = 0; auto &mo = ConstructorStats::get<MoveOnlyInt>(); - mo.move_assignments = mo.move_constructions = mo.copy_assignments = mo.copy_constructions = 0; + mo.move_assignments = mo.move_constructions = mo.copy_assignments = mo.copy_constructions + = 0; auto &co = ConstructorStats::get<CopyOnlyInt>(); - co.move_assignments = co.move_constructions = co.copy_assignments = co.copy_constructions = 0; + co.move_assignments = co.move_constructions = co.copy_assignments = co.copy_constructions + = 0; py::dict d; d["MoveOrCopyInt"] = py::cast(mc, py::return_value_policy::reference); d["MoveOnlyInt"] = py::cast(mo, py::return_value_policy::reference); @@ -159,18 +225,13 @@ TEST_SUBMODULE(copy_move_policies, m) { #ifdef PYBIND11_HAS_OPTIONAL // test_move_and_copy_load_optional m.attr("has_optional") = true; - m.def("move_optional", [](std::optional<MoveOnlyInt> o) { - return o->value; - }); - m.def("move_or_copy_optional", [](std::optional<MoveOrCopyInt> o) { - return o->value; - }); - m.def("copy_optional", [](std::optional<CopyOnlyInt> o) { - return o->value; - }); - m.def("move_optional_tuple", [](std::optional<std::tuple<MoveOrCopyInt, MoveOnlyInt, CopyOnlyInt>> x) { - return std::get<0>(*x).value + std::get<1>(*x).value + std::get<2>(*x).value; - }); + m.def("move_optional", [](std::optional<MoveOnlyInt> o) { return o->value; }); + m.def("move_or_copy_optional", [](std::optional<MoveOrCopyInt> o) { return o->value; }); + m.def("copy_optional", [](std::optional<CopyOnlyInt> o) { return o->value; }); + m.def("move_optional_tuple", + [](std::optional<std::tuple<MoveOrCopyInt, MoveOnlyInt, CopyOnlyInt>> x) { + return std::get<0>(*x).value + std::get<1>(*x).value + std::get<2>(*x).value; + }); #else m.attr("has_optional") = false; #endif @@ -181,40 +242,292 @@ TEST_SUBMODULE(copy_move_policies, m) { // added later. struct PrivateOpNew { int value = 1; + private: void *operator new(size_t bytes) { void *ptr = std::malloc(bytes); - if (ptr) + if (ptr) { return ptr; - else - throw std::bad_alloc{}; + } + throw std::bad_alloc{}; } }; py::class_<PrivateOpNew>(m, "PrivateOpNew").def_readonly("value", &PrivateOpNew::value); m.def("private_op_new_value", []() { return PrivateOpNew(); }); - m.def("private_op_new_reference", []() -> const PrivateOpNew & { - static PrivateOpNew x{}; - return x; - }, py::return_value_policy::reference); + m.def( + "private_op_new_reference", + []() -> const PrivateOpNew & { + static PrivateOpNew x{}; + return x; + }, + py::return_value_policy::reference); // test_move_fallback // #389: rvp::move should fall-through to copy on non-movable objects struct MoveIssue1 { int v; - MoveIssue1(int v) : v{v} {} + explicit MoveIssue1(int v) : v{v} {} MoveIssue1(const MoveIssue1 &c) = default; MoveIssue1(MoveIssue1 &&) = delete; }; - py::class_<MoveIssue1>(m, "MoveIssue1").def(py::init<int>()).def_readwrite("value", &MoveIssue1::v); + py::class_<MoveIssue1>(m, "MoveIssue1") + .def(py::init<int>()) + .def_readwrite("value", &MoveIssue1::v); struct MoveIssue2 { int v; - MoveIssue2(int v) : v{v} {} + explicit MoveIssue2(int v) : v{v} {} MoveIssue2(MoveIssue2 &&) = default; }; - py::class_<MoveIssue2>(m, "MoveIssue2").def(py::init<int>()).def_readwrite("value", &MoveIssue2::v); + py::class_<MoveIssue2>(m, "MoveIssue2") + .def(py::init<int>()) + .def_readwrite("value", &MoveIssue2::v); - // #2742: Don't expect ownership of raw pointer to `new`ed object to be transferred with `py::return_value_policy::move` - m.def("get_moveissue1", [](int i) { return std::unique_ptr<MoveIssue1>(new MoveIssue1(i)); }, py::return_value_policy::move); - m.def("get_moveissue2", [](int i) { return MoveIssue2(i); }, py::return_value_policy::move); + // #2742: Don't expect ownership of raw pointer to `new`ed object to be transferred with + // `py::return_value_policy::move` + m.def( + "get_moveissue1", + [](int i) { return std::unique_ptr<MoveIssue1>(new MoveIssue1(i)); }, + py::return_value_policy::move); + m.def( + "get_moveissue2", [](int i) { return MoveIssue2(i); }, py::return_value_policy::move); + + // Make sure that cast from pytype rvalue to other pytype works + m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast<py::int_>(); }); } + +/* + * Rest of the file: + * static_assert based tests for pybind11 adaptations of + * std::is_move_constructible, std::is_copy_constructible and + * std::is_copy_assignable (no adaptation of std::is_move_assignable). + * Difference between pybind11 and std traits: pybind11 traits will also check + * the contained value_types. + */ + +struct NotMovable { + NotMovable() = default; + NotMovable(NotMovable const &) = default; + NotMovable(NotMovable &&) = delete; + NotMovable &operator=(NotMovable const &) = default; + NotMovable &operator=(NotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible<NotMovable>::value, + "!std::is_move_constructible<NotMovable>::value"); +static_assert(std::is_copy_constructible<NotMovable>::value, + "std::is_copy_constructible<NotMovable>::value"); +static_assert(!pybind11::detail::is_move_constructible<NotMovable>::value, + "!pybind11::detail::is_move_constructible<NotMovable>::value"); +static_assert(pybind11::detail::is_copy_constructible<NotMovable>::value, + "pybind11::detail::is_copy_constructible<NotMovable>::value"); +static_assert(!std::is_move_assignable<NotMovable>::value, + "!std::is_move_assignable<NotMovable>::value"); +static_assert(std::is_copy_assignable<NotMovable>::value, + "std::is_copy_assignable<NotMovable>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotMovable>::value, +// "!pybind11::detail::is_move_assignable<NotMovable>::value"); +static_assert(pybind11::detail::is_copy_assignable<NotMovable>::value, + "pybind11::detail::is_copy_assignable<NotMovable>::value"); + +struct NotCopyable { + NotCopyable() = default; + NotCopyable(NotCopyable const &) = delete; + NotCopyable(NotCopyable &&) = default; + NotCopyable &operator=(NotCopyable const &) = delete; + NotCopyable &operator=(NotCopyable &&) = default; +}; +static_assert(std::is_move_constructible<NotCopyable>::value, + "std::is_move_constructible<NotCopyable>::value"); +static_assert(!std::is_copy_constructible<NotCopyable>::value, + "!std::is_copy_constructible<NotCopyable>::value"); +static_assert(pybind11::detail::is_move_constructible<NotCopyable>::value, + "pybind11::detail::is_move_constructible<NotCopyable>::value"); +static_assert(!pybind11::detail::is_copy_constructible<NotCopyable>::value, + "!pybind11::detail::is_copy_constructible<NotCopyable>::value"); +static_assert(std::is_move_assignable<NotCopyable>::value, + "std::is_move_assignable<NotCopyable>::value"); +static_assert(!std::is_copy_assignable<NotCopyable>::value, + "!std::is_copy_assignable<NotCopyable>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotCopyable>::value, +// "!pybind11::detail::is_move_assignable<NotCopyable>::value"); +static_assert(!pybind11::detail::is_copy_assignable<NotCopyable>::value, + "!pybind11::detail::is_copy_assignable<NotCopyable>::value"); + +struct NotCopyableNotMovable { + NotCopyableNotMovable() = default; + NotCopyableNotMovable(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable(NotCopyableNotMovable &&) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible<NotCopyableNotMovable>::value, + "!std::is_move_constructible<NotCopyableNotMovable>::value"); +static_assert(!std::is_copy_constructible<NotCopyableNotMovable>::value, + "!std::is_copy_constructible<NotCopyableNotMovable>::value"); +static_assert(!pybind11::detail::is_move_constructible<NotCopyableNotMovable>::value, + "!pybind11::detail::is_move_constructible<NotCopyableNotMovable>::value"); +static_assert(!pybind11::detail::is_copy_constructible<NotCopyableNotMovable>::value, + "!pybind11::detail::is_copy_constructible<NotCopyableNotMovable>::value"); +static_assert(!std::is_move_assignable<NotCopyableNotMovable>::value, + "!std::is_move_assignable<NotCopyableNotMovable>::value"); +static_assert(!std::is_copy_assignable<NotCopyableNotMovable>::value, + "!std::is_copy_assignable<NotCopyableNotMovable>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotCopyableNotMovable>::value, +// "!pybind11::detail::is_move_assignable<NotCopyableNotMovable>::value"); +static_assert(!pybind11::detail::is_copy_assignable<NotCopyableNotMovable>::value, + "!pybind11::detail::is_copy_assignable<NotCopyableNotMovable>::value"); + +struct NotMovableVector : std::vector<NotMovable> {}; +static_assert(std::is_move_constructible<NotMovableVector>::value, + "std::is_move_constructible<NotMovableVector>::value"); +static_assert(std::is_copy_constructible<NotMovableVector>::value, + "std::is_copy_constructible<NotMovableVector>::value"); +static_assert(!pybind11::detail::is_move_constructible<NotMovableVector>::value, + "!pybind11::detail::is_move_constructible<NotMovableVector>::value"); +static_assert(pybind11::detail::is_copy_constructible<NotMovableVector>::value, + "pybind11::detail::is_copy_constructible<NotMovableVector>::value"); +static_assert(std::is_move_assignable<NotMovableVector>::value, + "std::is_move_assignable<NotMovableVector>::value"); +static_assert(std::is_copy_assignable<NotMovableVector>::value, + "std::is_copy_assignable<NotMovableVector>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotMovableVector>::value, +// "!pybind11::detail::is_move_assignable<NotMovableVector>::value"); +static_assert(pybind11::detail::is_copy_assignable<NotMovableVector>::value, + "pybind11::detail::is_copy_assignable<NotMovableVector>::value"); + +struct NotCopyableVector : std::vector<NotCopyable> {}; +static_assert(std::is_move_constructible<NotCopyableVector>::value, + "std::is_move_constructible<NotCopyableVector>::value"); +static_assert(std::is_copy_constructible<NotCopyableVector>::value, + "std::is_copy_constructible<NotCopyableVector>::value"); +static_assert(pybind11::detail::is_move_constructible<NotCopyableVector>::value, + "pybind11::detail::is_move_constructible<NotCopyableVector>::value"); +static_assert(!pybind11::detail::is_copy_constructible<NotCopyableVector>::value, + "!pybind11::detail::is_copy_constructible<NotCopyableVector>::value"); +static_assert(std::is_move_assignable<NotCopyableVector>::value, + "std::is_move_assignable<NotCopyableVector>::value"); +static_assert(std::is_copy_assignable<NotCopyableVector>::value, + "std::is_copy_assignable<NotCopyableVector>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotCopyableVector>::value, +// "!pybind11::detail::is_move_assignable<NotCopyableVector>::value"); +static_assert(!pybind11::detail::is_copy_assignable<NotCopyableVector>::value, + "!pybind11::detail::is_copy_assignable<NotCopyableVector>::value"); + +struct NotCopyableNotMovableVector : std::vector<NotCopyableNotMovable> {}; +static_assert(std::is_move_constructible<NotCopyableNotMovableVector>::value, + "std::is_move_constructible<NotCopyableNotMovableVector>::value"); +static_assert(std::is_copy_constructible<NotCopyableNotMovableVector>::value, + "std::is_copy_constructible<NotCopyableNotMovableVector>::value"); +static_assert(!pybind11::detail::is_move_constructible<NotCopyableNotMovableVector>::value, + "!pybind11::detail::is_move_constructible<NotCopyableNotMovableVector>::value"); +static_assert(!pybind11::detail::is_copy_constructible<NotCopyableNotMovableVector>::value, + "!pybind11::detail::is_copy_constructible<NotCopyableNotMovableVector>::value"); +static_assert(std::is_move_assignable<NotCopyableNotMovableVector>::value, + "std::is_move_assignable<NotCopyableNotMovableVector>::value"); +static_assert(std::is_copy_assignable<NotCopyableNotMovableVector>::value, + "std::is_copy_assignable<NotCopyableNotMovableVector>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotCopyableNotMovableVector>::value, +// "!pybind11::detail::is_move_assignable<NotCopyableNotMovableVector>::value"); +static_assert(!pybind11::detail::is_copy_assignable<NotCopyableNotMovableVector>::value, + "!pybind11::detail::is_copy_assignable<NotCopyableNotMovableVector>::value"); + +struct NotMovableMap : std::map<int, NotMovable> {}; +static_assert(std::is_move_constructible<NotMovableMap>::value, + "std::is_move_constructible<NotMovableMap>::value"); +static_assert(std::is_copy_constructible<NotMovableMap>::value, + "std::is_copy_constructible<NotMovableMap>::value"); +static_assert(!pybind11::detail::is_move_constructible<NotMovableMap>::value, + "!pybind11::detail::is_move_constructible<NotMovableMap>::value"); +static_assert(pybind11::detail::is_copy_constructible<NotMovableMap>::value, + "pybind11::detail::is_copy_constructible<NotMovableMap>::value"); +static_assert(std::is_move_assignable<NotMovableMap>::value, + "std::is_move_assignable<NotMovableMap>::value"); +static_assert(std::is_copy_assignable<NotMovableMap>::value, + "std::is_copy_assignable<NotMovableMap>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotMovableMap>::value, +// "!pybind11::detail::is_move_assignable<NotMovableMap>::value"); +static_assert(pybind11::detail::is_copy_assignable<NotMovableMap>::value, + "pybind11::detail::is_copy_assignable<NotMovableMap>::value"); + +struct NotCopyableMap : std::map<int, NotCopyable> {}; +static_assert(std::is_move_constructible<NotCopyableMap>::value, + "std::is_move_constructible<NotCopyableMap>::value"); +static_assert(std::is_copy_constructible<NotCopyableMap>::value, + "std::is_copy_constructible<NotCopyableMap>::value"); +static_assert(pybind11::detail::is_move_constructible<NotCopyableMap>::value, + "pybind11::detail::is_move_constructible<NotCopyableMap>::value"); +static_assert(!pybind11::detail::is_copy_constructible<NotCopyableMap>::value, + "!pybind11::detail::is_copy_constructible<NotCopyableMap>::value"); +static_assert(std::is_move_assignable<NotCopyableMap>::value, + "std::is_move_assignable<NotCopyableMap>::value"); +static_assert(std::is_copy_assignable<NotCopyableMap>::value, + "std::is_copy_assignable<NotCopyableMap>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotCopyableMap>::value, +// "!pybind11::detail::is_move_assignable<NotCopyableMap>::value"); +static_assert(!pybind11::detail::is_copy_assignable<NotCopyableMap>::value, + "!pybind11::detail::is_copy_assignable<NotCopyableMap>::value"); + +struct NotCopyableNotMovableMap : std::map<int, NotCopyableNotMovable> {}; +static_assert(std::is_move_constructible<NotCopyableNotMovableMap>::value, + "std::is_move_constructible<NotCopyableNotMovableMap>::value"); +static_assert(std::is_copy_constructible<NotCopyableNotMovableMap>::value, + "std::is_copy_constructible<NotCopyableNotMovableMap>::value"); +static_assert(!pybind11::detail::is_move_constructible<NotCopyableNotMovableMap>::value, + "!pybind11::detail::is_move_constructible<NotCopyableNotMovableMap>::value"); +static_assert(!pybind11::detail::is_copy_constructible<NotCopyableNotMovableMap>::value, + "!pybind11::detail::is_copy_constructible<NotCopyableNotMovableMap>::value"); +static_assert(std::is_move_assignable<NotCopyableNotMovableMap>::value, + "std::is_move_assignable<NotCopyableNotMovableMap>::value"); +static_assert(std::is_copy_assignable<NotCopyableNotMovableMap>::value, + "std::is_copy_assignable<NotCopyableNotMovableMap>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<NotCopyableNotMovableMap>::value, +// "!pybind11::detail::is_move_assignable<NotCopyableNotMovableMap>::value"); +static_assert(!pybind11::detail::is_copy_assignable<NotCopyableNotMovableMap>::value, + "!pybind11::detail::is_copy_assignable<NotCopyableNotMovableMap>::value"); + +struct RecursiveVector : std::vector<RecursiveVector> {}; +static_assert(std::is_move_constructible<RecursiveVector>::value, + "std::is_move_constructible<RecursiveVector>::value"); +static_assert(std::is_copy_constructible<RecursiveVector>::value, + "std::is_copy_constructible<RecursiveVector>::value"); +static_assert(pybind11::detail::is_move_constructible<RecursiveVector>::value, + "pybind11::detail::is_move_constructible<RecursiveVector>::value"); +static_assert(pybind11::detail::is_copy_constructible<RecursiveVector>::value, + "pybind11::detail::is_copy_constructible<RecursiveVector>::value"); +static_assert(std::is_move_assignable<RecursiveVector>::value, + "std::is_move_assignable<RecursiveVector>::value"); +static_assert(std::is_copy_assignable<RecursiveVector>::value, + "std::is_copy_assignable<RecursiveVector>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<RecursiveVector>::value, +// "!pybind11::detail::is_move_assignable<RecursiveVector>::value"); +static_assert(pybind11::detail::is_copy_assignable<RecursiveVector>::value, + "pybind11::detail::is_copy_assignable<RecursiveVector>::value"); + +struct RecursiveMap : std::map<int, RecursiveMap> {}; +static_assert(std::is_move_constructible<RecursiveMap>::value, + "std::is_move_constructible<RecursiveMap>::value"); +static_assert(std::is_copy_constructible<RecursiveMap>::value, + "std::is_copy_constructible<RecursiveMap>::value"); +static_assert(pybind11::detail::is_move_constructible<RecursiveMap>::value, + "pybind11::detail::is_move_constructible<RecursiveMap>::value"); +static_assert(pybind11::detail::is_copy_constructible<RecursiveMap>::value, + "pybind11::detail::is_copy_constructible<RecursiveMap>::value"); +static_assert(std::is_move_assignable<RecursiveMap>::value, + "std::is_move_assignable<RecursiveMap>::value"); +static_assert(std::is_copy_assignable<RecursiveMap>::value, + "std::is_copy_assignable<RecursiveMap>::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable<RecursiveMap>::value, +// "!pybind11::detail::is_move_assignable<RecursiveMap>::value"); +static_assert(pybind11::detail::is_copy_assignable<RecursiveMap>::value, + "pybind11::detail::is_copy_assignable<RecursiveMap>::value"); diff --git a/tests/test_copy_move.py b/tests/test_copy_move.py index 1d989522..9fef0893 100644 --- a/tests/test_copy_move.py +++ b/tests/test_copy_move.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- import pytest + from pybind11_tests import copy_move_policies as m @@ -123,3 +123,10 @@ def test_move_fallback(): assert m1.value == 1 m2 = m.get_moveissue2(2) assert m2.value == 2 + + +def test_pytype_rvalue_cast(): + """Make sure that cast from pytype rvalue to other pytype works""" + + value = m.get_pytype_rvalue_castissue(1.0) + assert value == 1 diff --git a/tests/test_custom_type_casters.cpp b/tests/test_custom_type_casters.cpp index 3fe910d4..b4af02a4 100644 --- a/tests/test_custom_type_casters.cpp +++ b/tests/test_custom_type_casters.cpp @@ -7,23 +7,37 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include "constructor_stats.h" - +#include "pybind11_tests.h" // py::arg/py::arg_v testing: these arguments just record their argument when invoked -class ArgInspector1 { public: std::string arg = "(default arg inspector 1)"; }; -class ArgInspector2 { public: std::string arg = "(default arg inspector 2)"; }; -class ArgAlwaysConverts { }; -namespace pybind11 { namespace detail { -template <> struct type_caster<ArgInspector1> { +class ArgInspector1 { +public: + std::string arg = "(default arg inspector 1)"; +}; +class ArgInspector2 { +public: + std::string arg = "(default arg inspector 2)"; +}; +class ArgAlwaysConverts {}; + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster<ArgInspector1> { public: + // Classic +#ifdef PYBIND11_DETAIL_UNDERSCORE_BACKWARD_COMPATIBILITY PYBIND11_TYPE_CASTER(ArgInspector1, _("ArgInspector1")); +#else + PYBIND11_TYPE_CASTER(ArgInspector1, const_name("ArgInspector1")); +#endif bool load(handle src, bool convert) { - value.arg = "loading ArgInspector1 argument " + - std::string(convert ? "WITH" : "WITHOUT") + " conversion allowed. " - "Argument value = " + (std::string) str(src); + value.arg = "loading ArgInspector1 argument " + std::string(convert ? "WITH" : "WITHOUT") + + " conversion allowed. " + "Argument value = " + + (std::string) str(src); return true; } @@ -31,14 +45,16 @@ public: return str(src.arg).release(); } }; -template <> struct type_caster<ArgInspector2> { +template <> +struct type_caster<ArgInspector2> { public: - PYBIND11_TYPE_CASTER(ArgInspector2, _("ArgInspector2")); + PYBIND11_TYPE_CASTER(ArgInspector2, const_name("ArgInspector2")); bool load(handle src, bool convert) { - value.arg = "loading ArgInspector2 argument " + - std::string(convert ? "WITH" : "WITHOUT") + " conversion allowed. " - "Argument value = " + (std::string) str(src); + value.arg = "loading ArgInspector2 argument " + std::string(convert ? "WITH" : "WITHOUT") + + " conversion allowed. " + "Argument value = " + + (std::string) str(src); return true; } @@ -46,20 +62,19 @@ public: return str(src.arg).release(); } }; -template <> struct type_caster<ArgAlwaysConverts> { +template <> +struct type_caster<ArgAlwaysConverts> { public: - PYBIND11_TYPE_CASTER(ArgAlwaysConverts, _("ArgAlwaysConverts")); + PYBIND11_TYPE_CASTER(ArgAlwaysConverts, const_name("ArgAlwaysConverts")); - bool load(handle, bool convert) { - return convert; - } + bool load(handle, bool convert) { return convert; } static handle cast(const ArgAlwaysConverts &, return_value_policy, handle) { return py::none().release(); } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // test_custom_caster_destruction class DestructionTester { @@ -67,13 +82,21 @@ public: DestructionTester() { print_default_created(this); } ~DestructionTester() { print_destroyed(this); } DestructionTester(const DestructionTester &) { print_copy_created(this); } - DestructionTester(DestructionTester &&) { print_move_created(this); } - DestructionTester &operator=(const DestructionTester &) { print_copy_assigned(this); return *this; } - DestructionTester &operator=(DestructionTester &&) { print_move_assigned(this); return *this; } + DestructionTester(DestructionTester &&) noexcept { print_move_created(this); } + DestructionTester &operator=(const DestructionTester &) { + print_copy_assigned(this); + return *this; + } + DestructionTester &operator=(DestructionTester &&) noexcept { + print_move_assigned(this); + return *this; + } }; -namespace pybind11 { namespace detail { -template <> struct type_caster<DestructionTester> { - PYBIND11_TYPE_CASTER(DestructionTester, _("DestructionTester")); +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster<DestructionTester> { + PYBIND11_TYPE_CASTER(DestructionTester, const_name("DestructionTester")); bool load(handle, bool) { return true; } static handle cast(const DestructionTester &, return_value_policy, handle) { @@ -81,7 +104,35 @@ template <> struct type_caster<DestructionTester> { } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE + +// Define type caster outside of `pybind11::detail` and then alias it. +namespace other_lib { +struct MyType {}; +// Corrupt `py` shorthand alias for surrounding context. +namespace py {} +// Corrupt unqualified relative `pybind11` namespace. +namespace PYBIND11_NAMESPACE {} +// Correct alias. +namespace py_ = ::pybind11; +// Define caster. This is effectively no-op, we only ensure it compiles and we +// don't have any symbol collision when using macro mixin. +struct my_caster { + PYBIND11_TYPE_CASTER(MyType, py_::detail::const_name("MyType")); + bool load(py_::handle, bool) { return true; } + + static py_::handle cast(const MyType &, py_::return_value_policy, py_::handle) { + return py_::bool_(true).release(); + } +}; +} // namespace other_lib +// Effectively "alias" it into correct namespace (via inheritance). +namespace PYBIND11_NAMESPACE { +namespace detail { +template <> +struct type_caster<other_lib::MyType> : public other_lib::my_caster {}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(custom_type_casters, m) { // test_custom_type_casters @@ -94,7 +145,11 @@ TEST_SUBMODULE(custom_type_casters, m) { class ArgInspector { public: ArgInspector1 f(ArgInspector1 a, ArgAlwaysConverts) { return a; } - std::string g(ArgInspector1 a, const ArgInspector1 &b, int c, ArgInspector2 *d, ArgAlwaysConverts) { + std::string g(const ArgInspector1 &a, + const ArgInspector1 &b, + int c, + ArgInspector2 *d, + ArgAlwaysConverts) { return a.arg + "\n" + b.arg + "\n" + std::to_string(c) + "\n" + d->arg; } static ArgInspector2 h(ArgInspector2 a, ArgAlwaysConverts) { return a; } @@ -103,26 +158,52 @@ TEST_SUBMODULE(custom_type_casters, m) { py::class_<ArgInspector>(m, "ArgInspector") .def(py::init<>()) .def("f", &ArgInspector::f, py::arg(), py::arg() = ArgAlwaysConverts()) - .def("g", &ArgInspector::g, "a"_a.noconvert(), "b"_a, "c"_a.noconvert()=13, "d"_a=ArgInspector2(), py::arg() = ArgAlwaysConverts()) - .def_static("h", &ArgInspector::h, py::arg{}.noconvert(), py::arg() = ArgAlwaysConverts()) - ; - m.def("arg_inspect_func", [](ArgInspector2 a, ArgInspector1 b, ArgAlwaysConverts) { return a.arg + "\n" + b.arg; }, - py::arg{}.noconvert(false), py::arg_v(nullptr, ArgInspector1()).noconvert(true), py::arg() = ArgAlwaysConverts()); - - m.def("floats_preferred", [](double f) { return 0.5 * f; }, "f"_a); - m.def("floats_only", [](double f) { return 0.5 * f; }, "f"_a.noconvert()); - m.def("ints_preferred", [](int i) { return i / 2; }, "i"_a); - m.def("ints_only", [](int i) { return i / 2; }, "i"_a.noconvert()); + .def("g", + &ArgInspector::g, + "a"_a.noconvert(), + "b"_a, + "c"_a.noconvert() = 13, + "d"_a = ArgInspector2(), + py::arg() = ArgAlwaysConverts()) + .def_static("h", &ArgInspector::h, py::arg{}.noconvert(), py::arg() = ArgAlwaysConverts()); + m.def( + "arg_inspect_func", + [](const ArgInspector2 &a, const ArgInspector1 &b, ArgAlwaysConverts) { + return a.arg + "\n" + b.arg; + }, + py::arg{}.noconvert(false), + py::arg_v(nullptr, ArgInspector1()).noconvert(true), + py::arg() = ArgAlwaysConverts()); + + m.def( + "floats_preferred", [](double f) { return 0.5 * f; }, "f"_a); + m.def( + "floats_only", [](double f) { return 0.5 * f; }, "f"_a.noconvert()); + m.def( + "ints_preferred", [](int i) { return i / 2; }, "i"_a); + m.def( + "ints_only", [](int i) { return i / 2; }, "i"_a.noconvert()); // test_custom_caster_destruction // Test that `take_ownership` works on types with a custom type caster when given a pointer // default policy: don't take ownership: - m.def("custom_caster_no_destroy", []() { static auto *dt = new DestructionTester(); return dt; }); - - m.def("custom_caster_destroy", []() { return new DestructionTester(); }, - py::return_value_policy::take_ownership); // Takes ownership: destroy when finished - m.def("custom_caster_destroy_const", []() -> const DestructionTester * { return new DestructionTester(); }, - py::return_value_policy::take_ownership); // Likewise (const doesn't inhibit destruction) - m.def("destruction_tester_cstats", &ConstructorStats::get<DestructionTester>, py::return_value_policy::reference); + m.def("custom_caster_no_destroy", []() { + static auto *dt = new DestructionTester(); + return dt; + }); + + m.def( + "custom_caster_destroy", + []() { return new DestructionTester(); }, + py::return_value_policy::take_ownership); // Takes ownership: destroy when finished + m.def( + "custom_caster_destroy_const", + []() -> const DestructionTester * { return new DestructionTester(); }, + py::return_value_policy::take_ownership); // Likewise (const doesn't inhibit destruction) + m.def("destruction_tester_cstats", + &ConstructorStats::get<DestructionTester>, + py::return_value_policy::reference); + + m.def("other_lib_type", [](other_lib::MyType x) { return x; }); } diff --git a/tests/test_custom_type_casters.py b/tests/test_custom_type_casters.py index bb74d54e..3a00ea96 100644 --- a/tests/test_custom_type_casters.py +++ b/tests/test_custom_type_casters.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- import pytest + from pybind11_tests import custom_type_casters as m @@ -18,7 +18,7 @@ def test_noconvert_args(msg): loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b 13 loading ArgInspector2 argument WITH conversion allowed. Argument value = (default arg inspector 2) - """ # noqa: E501 line too long + """ ) assert ( msg(a.g("this is a", "this is b", 42)) @@ -27,7 +27,7 @@ def test_noconvert_args(msg): loading ArgInspector1 argument WITH conversion allowed. Argument value = this is b 42 loading ArgInspector2 argument WITH conversion allowed. Argument value = (default arg inspector 2) - """ # noqa: E501 line too long + """ ) assert ( msg(a.g("this is a", "this is b", 42, "this is d")) @@ -75,7 +75,7 @@ def test_noconvert_args(msg): 1. (i: int) -> int Invoked with: 4.0 - """ # noqa: E501 line too long + """ ) assert m.ints_only(4) == 2 @@ -94,12 +94,14 @@ def test_noconvert_args(msg): def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets - destroyed when the function has py::return_value_policy::take_ownership policy applied.""" + destroyed when the function has py::return_value_policy::take_ownership policy applied. + """ cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: z = m.custom_caster_no_destroy() - assert cstats.alive() == 1 and cstats.default_constructions == 1 + assert cstats.alive() == 1 + assert cstats.default_constructions == 1 assert z # take_ownership applied: this constructs a new object, casts it, then destroys it: @@ -114,3 +116,7 @@ def test_custom_caster_destruction(): # Make sure we still only have the original object (from ..._no_destroy()) alive: assert cstats.alive() == 1 + + +def test_custom_caster_other_lib(): + assert m.other_lib_type(True) diff --git a/tests/test_custom_type_setup.cpp b/tests/test_custom_type_setup.cpp new file mode 100644 index 00000000..42fae05d --- /dev/null +++ b/tests/test_custom_type_setup.cpp @@ -0,0 +1,41 @@ +/* + tests/test_custom_type_setup.cpp -- Tests `pybind11::custom_type_setup` + + Copyright (c) Google LLC + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include <pybind11/pybind11.h> + +#include "pybind11_tests.h" + +namespace py = pybind11; + +namespace { + +struct OwnsPythonObjects { + py::object value = py::none(); +}; +} // namespace + +TEST_SUBMODULE(custom_type_setup, m) { + py::class_<OwnsPythonObjects> cls( + m, "OwnsPythonObjects", py::custom_type_setup([](PyHeapTypeObject *heap_type) { + auto *type = &heap_type->ht_type; + type->tp_flags |= Py_TPFLAGS_HAVE_GC; + type->tp_traverse = [](PyObject *self_base, visitproc visit, void *arg) { + auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base)); + Py_VISIT(self.value.ptr()); + return 0; + }; + type->tp_clear = [](PyObject *self_base) { + auto &self = py::cast<OwnsPythonObjects &>(py::handle(self_base)); + self.value = py::none(); + return 0; + }; + })); + cls.def(py::init<>()); + cls.def_readwrite("value", &OwnsPythonObjects::value); +} diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py new file mode 100644 index 00000000..e63ff575 --- /dev/null +++ b/tests/test_custom_type_setup.py @@ -0,0 +1,48 @@ +import gc +import weakref + +import pytest + +import env # noqa: F401 +from pybind11_tests import custom_type_setup as m + + +@pytest.fixture() +def gc_tester(): + """Tests that an object is garbage collected. + + Assumes that any unreferenced objects are fully collected after calling + `gc.collect()`. That is true on CPython, but does not appear to reliably + hold on PyPy. + """ + + weak_refs = [] + + def add_ref(obj): + # PyPy does not support `gc.is_tracked`. + if hasattr(gc, "is_tracked"): + assert gc.is_tracked(obj) + weak_refs.append(weakref.ref(obj)) + + yield add_ref + + gc.collect() + for ref in weak_refs: + assert ref() is None + + +# PyPy does not seem to reliably garbage collect. +@pytest.mark.skipif("env.PYPY") +def test_self_cycle(gc_tester): + obj = m.OwnsPythonObjects() + obj.value = obj + gc_tester(obj) + + +# PyPy does not seem to reliably garbage collect. +@pytest.mark.skipif("env.PYPY") +def test_indirect_cycle(gc_tester): + obj = m.OwnsPythonObjects() + obj_list = [obj] + obj.value = obj_list + gc_tester(obj) diff --git a/tests/test_docstring_options.cpp b/tests/test_docstring_options.cpp index 8a97af55..dda1cf6e 100644 --- a/tests/test_docstring_options.cpp +++ b/tests/test_docstring_options.cpp @@ -15,35 +15,52 @@ TEST_SUBMODULE(docstring_options, m) { py::options options; options.disable_function_signatures(); - m.def("test_function1", [](int, int) {}, py::arg("a"), py::arg("b")); - m.def("test_function2", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); - - m.def("test_overloaded1", [](int) {}, py::arg("i"), "Overload docstring"); - m.def("test_overloaded1", [](double) {}, py::arg("d")); - - m.def("test_overloaded2", [](int) {}, py::arg("i"), "overload docstring 1"); - m.def("test_overloaded2", [](double) {}, py::arg("d"), "overload docstring 2"); - - m.def("test_overloaded3", [](int) {}, py::arg("i")); - m.def("test_overloaded3", [](double) {}, py::arg("d"), "Overload docstr"); + m.def( + "test_function1", [](int, int) {}, py::arg("a"), py::arg("b")); + m.def( + "test_function2", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + + m.def( + "test_overloaded1", [](int) {}, py::arg("i"), "Overload docstring"); + m.def( + "test_overloaded1", [](double) {}, py::arg("d")); + + m.def( + "test_overloaded2", [](int) {}, py::arg("i"), "overload docstring 1"); + m.def( + "test_overloaded2", [](double) {}, py::arg("d"), "overload docstring 2"); + + m.def( + "test_overloaded3", [](int) {}, py::arg("i")); + m.def( + "test_overloaded3", [](double) {}, py::arg("d"), "Overload docstr"); options.enable_function_signatures(); - m.def("test_function3", [](int, int) {}, py::arg("a"), py::arg("b")); - m.def("test_function4", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + m.def( + "test_function3", [](int, int) {}, py::arg("a"), py::arg("b")); + m.def( + "test_function4", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); options.disable_function_signatures().disable_user_defined_docstrings(); - m.def("test_function5", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + m.def( + "test_function5", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); { py::options nested_options; nested_options.enable_user_defined_docstrings(); - m.def("test_function6", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + m.def( + "test_function6", + [](int, int) {}, + py::arg("a"), + py::arg("b"), + "A custom docstring"); } } - m.def("test_function7", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); + m.def( + "test_function7", [](int, int) {}, py::arg("a"), py::arg("b"), "A custom docstring"); { py::options options; @@ -63,7 +80,62 @@ TEST_SUBMODULE(docstring_options, m) { int getValue() const { return value; } }; py::class_<DocstringTestFoo>(m, "DocstringTestFoo", "This is a class docstring") - .def_property("value_prop", &DocstringTestFoo::getValue, &DocstringTestFoo::setValue, "This is a property docstring") - ; + .def_property("value_prop", + &DocstringTestFoo::getValue, + &DocstringTestFoo::setValue, + "This is a property docstring"); + } + + { + enum class DocstringTestEnum1 { Member1, Member2 }; + + py::enum_<DocstringTestEnum1>(m, "DocstringTestEnum1", "Enum docstring") + .value("Member1", DocstringTestEnum1::Member1) + .value("Member2", DocstringTestEnum1::Member2); + } + + { + py::options options; + options.enable_enum_members_docstring(); + + enum class DocstringTestEnum2 { Member1, Member2 }; + + py::enum_<DocstringTestEnum2>(m, "DocstringTestEnum2", "Enum docstring") + .value("Member1", DocstringTestEnum2::Member1) + .value("Member2", DocstringTestEnum2::Member2); + } + + { + py::options options; + options.disable_enum_members_docstring(); + + enum class DocstringTestEnum3 { Member1, Member2 }; + + py::enum_<DocstringTestEnum3>(m, "DocstringTestEnum3", "Enum docstring") + .value("Member1", DocstringTestEnum3::Member1) + .value("Member2", DocstringTestEnum3::Member2); + } + + { + py::options options; + options.disable_user_defined_docstrings(); + + enum class DocstringTestEnum4 { Member1, Member2 }; + + py::enum_<DocstringTestEnum4>(m, "DocstringTestEnum4", "Enum docstring") + .value("Member1", DocstringTestEnum4::Member1) + .value("Member2", DocstringTestEnum4::Member2); + } + + { + py::options options; + options.disable_user_defined_docstrings(); + options.disable_enum_members_docstring(); + + enum class DocstringTestEnum5 { Member1, Member2 }; + + py::enum_<DocstringTestEnum5>(m, "DocstringTestEnum5", "Enum docstring") + .value("Member1", DocstringTestEnum5::Member1) + .value("Member2", DocstringTestEnum5::Member2); } } diff --git a/tests/test_docstring_options.py b/tests/test_docstring_options.py index 8ee66138..e6f5a9d9 100644 --- a/tests/test_docstring_options.py +++ b/tests/test_docstring_options.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from pybind11_tests import docstring_options as m @@ -40,3 +39,26 @@ def test_docstring_options(): # Suppression of user-defined docstrings for non-function objects assert not m.DocstringTestFoo.__doc__ assert not m.DocstringTestFoo.value_prop.__doc__ + + # Check existig behaviour of enum docstings + assert ( + m.DocstringTestEnum1.__doc__ + == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" + ) + + # options.enable_enum_members_docstring() + assert ( + m.DocstringTestEnum2.__doc__ + == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" + ) + + # options.disable_enum_members_docstring() + assert m.DocstringTestEnum3.__doc__ == "Enum docstring" + + # options.disable_user_defined_docstrings() + assert m.DocstringTestEnum4.__doc__ == "Members:\n\n Member1\n\n Member2" + + # options.disable_user_defined_docstrings() + # options.disable_enum_members_docstring() + # When all options are disabled, no docstring (instead of an empty one) should be generated + assert m.DocstringTestEnum5.__doc__ is None diff --git a/tests/test_eigen.cpp b/tests/test_eigen_matrix.cpp index 84325474..554cc4d7 100644 --- a/tests/test_eigen.cpp +++ b/tests/test_eigen_matrix.cpp @@ -7,26 +7,27 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" -#include <pybind11/eigen.h> +#include <pybind11/eigen/matrix.h> #include <pybind11/stl.h> -#if defined(_MSC_VER) -# pragma warning(disable: 4996) // C4996: std::unary_negation is deprecated -#endif +#include "constructor_stats.h" +#include "pybind11_tests.h" + +PYBIND11_WARNING_DISABLE_MSVC(4996) #include <Eigen/Cholesky> using MatrixXdR = Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>; - - // Sets/resets a testing reference matrix to have values of 10*r + c, where r and c are the // (1-based) row/column number. -template <typename M> void reset_ref(M &x) { - for (int i = 0; i < x.rows(); i++) for (int j = 0; j < x.cols(); j++) - x(i, j) = 11 + 10*i + j; +template <typename M> +void reset_ref(M &x) { + for (int i = 0; i < x.rows(); i++) { + for (int j = 0; j < x.cols(); j++) { + x(i, j) = 11 + 10 * i + j; + } + } } // Returns a static, column-major matrix @@ -54,16 +55,18 @@ void reset_refs() { } // Returns element 2,1 from a matrix (used to test copy/nocopy) -double get_elem(Eigen::Ref<const Eigen::MatrixXd> m) { return m(2, 1); }; - +double get_elem(const Eigen::Ref<const Eigen::MatrixXd> &m) { return m(2, 1); }; // Returns a matrix with 10*r + 100*c added to each matrix element (to help test that the matrix // reference is referencing rows/columns correctly). -template <typename MatrixArgType> Eigen::MatrixXd adjust_matrix(MatrixArgType m) { +template <typename MatrixArgType> +Eigen::MatrixXd adjust_matrix(MatrixArgType m) { Eigen::MatrixXd ret(m); - for (int c = 0; c < m.cols(); c++) - for (int r = 0; r < m.rows(); r++) - ret(r, c) += 10*r + 100*c; // NOLINT(clang-analyzer-core.uninitialized.Assign) + for (int c = 0; c < m.cols(); c++) { + for (int r = 0; r < m.rows(); r++) { + ret(r, c) += 10 * r + 100 * c; // NOLINT(clang-analyzer-core.uninitialized.Assign) + } + } return ret; } @@ -76,7 +79,7 @@ struct CustomOperatorNew { EIGEN_MAKE_ALIGNED_OPERATOR_NEW; }; -TEST_SUBMODULE(eigen, m) { +TEST_SUBMODULE(eigen_matrix, m) { using FixedMatrixR = Eigen::Matrix<float, 5, 6, Eigen::RowMajor>; using FixedMatrixC = Eigen::Matrix<float, 5, 6>; using DenseMatrixR = Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>; @@ -90,27 +93,35 @@ TEST_SUBMODULE(eigen, m) { // various tests m.def("double_col", [](const Eigen::VectorXf &x) -> Eigen::VectorXf { return 2.0f * x; }); - m.def("double_row", [](const Eigen::RowVectorXf &x) -> Eigen::RowVectorXf { return 2.0f * x; }); - m.def("double_complex", [](const Eigen::VectorXcf &x) -> Eigen::VectorXcf { return 2.0f * x; }); + m.def("double_row", + [](const Eigen::RowVectorXf &x) -> Eigen::RowVectorXf { return 2.0f * x; }); + m.def("double_complex", + [](const Eigen::VectorXcf &x) -> Eigen::VectorXcf { return 2.0f * x; }); m.def("double_threec", [](py::EigenDRef<Eigen::Vector3f> x) { x *= 2; }); m.def("double_threer", [](py::EigenDRef<Eigen::RowVector3f> x) { x *= 2; }); - m.def("double_mat_cm", [](Eigen::MatrixXf x) -> Eigen::MatrixXf { return 2.0f * x; }); - m.def("double_mat_rm", [](DenseMatrixR x) -> DenseMatrixR { return 2.0f * x; }); + m.def("double_mat_cm", [](const Eigen::MatrixXf &x) -> Eigen::MatrixXf { return 2.0f * x; }); + m.def("double_mat_rm", [](const DenseMatrixR &x) -> DenseMatrixR { return 2.0f * x; }); // test_eigen_ref_to_python // Different ways of passing via Eigen::Ref; the first and second are the Eigen-recommended - m.def("cholesky1", [](Eigen::Ref<MatrixXdR> x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); - m.def("cholesky2", [](const Eigen::Ref<const MatrixXdR> &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); - m.def("cholesky3", [](const Eigen::Ref<MatrixXdR> &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); - m.def("cholesky4", [](Eigen::Ref<const MatrixXdR> x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky1", + [](const Eigen::Ref<MatrixXdR> &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky2", [](const Eigen::Ref<const MatrixXdR> &x) -> Eigen::MatrixXd { + return x.llt().matrixL(); + }); + m.def("cholesky3", + [](const Eigen::Ref<MatrixXdR> &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky4", [](const Eigen::Ref<const MatrixXdR> &x) -> Eigen::MatrixXd { + return x.llt().matrixL(); + }); // test_eigen_ref_mutators - // Mutators: these add some value to the given element using Eigen, but Eigen should be mapping into - // the numpy array data and so the result should show up there. There are three versions: one that - // works on a contiguous-row matrix (numpy's default), one for a contiguous-column matrix, and one - // for any matrix. - auto add_rm = [](Eigen::Ref<MatrixXdR> x, int r, int c, double v) { x(r,c) += v; }; - auto add_cm = [](Eigen::Ref<Eigen::MatrixXd> x, int r, int c, double v) { x(r,c) += v; }; + // Mutators: these add some value to the given element using Eigen, but Eigen should be mapping + // into the numpy array data and so the result should show up there. There are three versions: + // one that works on a contiguous-row matrix (numpy's default), one for a contiguous-column + // matrix, and one for any matrix. + auto add_rm = [](Eigen::Ref<MatrixXdR> x, int r, int c, double v) { x(r, c) += v; }; + auto add_cm = [](Eigen::Ref<Eigen::MatrixXd> x, int r, int c, double v) { x(r, c) += v; }; // Mutators (Eigen maps into numpy variables): m.def("add_rm", add_rm); // Only takes row-contiguous @@ -121,7 +132,8 @@ TEST_SUBMODULE(eigen, m) { m.def("add2", add_cm); m.def("add2", add_rm); // This one accepts a matrix of any stride: - m.def("add_any", [](py::EigenDRef<Eigen::MatrixXd> x, int r, int c, double v) { x(r,c) += v; }); + m.def("add_any", + [](py::EigenDRef<Eigen::MatrixXd> x, int r, int c, double v) { x(r, c) += v; }); // Return mutable references (numpy maps into eigen variables) m.def("get_cm_ref", []() { return Eigen::Ref<Eigen::MatrixXd>(get_cm()); }); @@ -133,49 +145,101 @@ TEST_SUBMODULE(eigen, m) { m.def("reset_refs", reset_refs); // Restores get_{cm,rm}_ref to original values // Increments and returns ref to (same) matrix - m.def("incr_matrix", [](Eigen::Ref<Eigen::MatrixXd> m, double v) { - m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); - return m; - }, py::return_value_policy::reference); + m.def( + "incr_matrix", + [](Eigen::Ref<Eigen::MatrixXd> m, double v) { + m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); + return m; + }, + py::return_value_policy::reference); // Same, but accepts a matrix of any strides - m.def("incr_matrix_any", [](py::EigenDRef<Eigen::MatrixXd> m, double v) { - m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); - return m; - }, py::return_value_policy::reference); + m.def( + "incr_matrix_any", + [](py::EigenDRef<Eigen::MatrixXd> m, double v) { + m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); + return m; + }, + py::return_value_policy::reference); // Returns an eigen slice of even rows - m.def("even_rows", [](py::EigenDRef<Eigen::MatrixXd> m) { - return py::EigenDMap<Eigen::MatrixXd>( - m.data(), (m.rows() + 1) / 2, m.cols(), + m.def( + "even_rows", + [](py::EigenDRef<Eigen::MatrixXd> m) { + return py::EigenDMap<Eigen::MatrixXd>( + m.data(), + (m.rows() + 1) / 2, + m.cols(), py::EigenDStride(m.outerStride(), 2 * m.innerStride())); - }, py::return_value_policy::reference); + }, + py::return_value_policy::reference); // Returns an eigen slice of even columns - m.def("even_cols", [](py::EigenDRef<Eigen::MatrixXd> m) { - return py::EigenDMap<Eigen::MatrixXd>( - m.data(), m.rows(), (m.cols() + 1) / 2, + m.def( + "even_cols", + [](py::EigenDRef<Eigen::MatrixXd> m) { + return py::EigenDMap<Eigen::MatrixXd>( + m.data(), + m.rows(), + (m.cols() + 1) / 2, py::EigenDStride(2 * m.outerStride(), m.innerStride())); - }, py::return_value_policy::reference); + }, + py::return_value_policy::reference); // Returns diagonals: a vector-like object with an inner stride != 1 m.def("diagonal", [](const Eigen::Ref<const Eigen::MatrixXd> &x) { return x.diagonal(); }); - m.def("diagonal_1", [](const Eigen::Ref<const Eigen::MatrixXd> &x) { return x.diagonal<1>(); }); - m.def("diagonal_n", [](const Eigen::Ref<const Eigen::MatrixXd> &x, int index) { return x.diagonal(index); }); + m.def("diagonal_1", + [](const Eigen::Ref<const Eigen::MatrixXd> &x) { return x.diagonal<1>(); }); + m.def("diagonal_n", + [](const Eigen::Ref<const Eigen::MatrixXd> &x, int index) { return x.diagonal(index); }); // Return a block of a matrix (gives non-standard strides) - m.def("block", [](const Eigen::Ref<const Eigen::MatrixXd> &x, int start_row, int start_col, int block_rows, int block_cols) { - return x.block(start_row, start_col, block_rows, block_cols); - }); + m.def("block", + [m](const py::object &x_obj, + int start_row, + int start_col, + int block_rows, + int block_cols) { + return m.attr("_block")(x_obj, x_obj, start_row, start_col, block_rows, block_cols); + }); + + m.def( + "_block", + [](const py::object &x_obj, + const Eigen::Ref<const Eigen::MatrixXd> &x, + int start_row, + int start_col, + int block_rows, + int block_cols) { + // See PR #4217 for background. This test is a bit over the top, but might be useful + // as a concrete example to point to when explaining the dangling reference trap. + auto i0 = py::make_tuple(0, 0); + auto x0_orig = x_obj[*i0].cast<double>(); + if (x(0, 0) != x0_orig) { + throw std::runtime_error( + "Something in the type_caster for Eigen::Ref is terribly wrong."); + } + double x0_mod = x0_orig + 1; + x_obj[*i0] = x0_mod; + auto copy_detected = (x(0, 0) != x0_mod); + x_obj[*i0] = x0_orig; + if (copy_detected) { + throw std::runtime_error("type_caster for Eigen::Ref made a copy."); + } + return x.block(start_row, start_col, block_rows, block_cols); + }, + py::keep_alive<0, 1>()); // test_eigen_return_references, test_eigen_keepalive // return value referencing/copying tests: class ReturnTester { Eigen::MatrixXd mat = create(); + public: ReturnTester() { print_created(this); } ~ReturnTester() { print_destroyed(this); } static Eigen::MatrixXd create() { return Eigen::MatrixXd::Ones(10, 10); } + // NOLINTNEXTLINE(readability-const-return-type) static const Eigen::MatrixXd createConst() { return Eigen::MatrixXd::Ones(10, 10); } Eigen::MatrixXd &get() { return mat; } Eigen::MatrixXd *getPtr() { return &mat; } @@ -183,12 +247,24 @@ TEST_SUBMODULE(eigen, m) { const Eigen::MatrixXd *viewPtr() { return &mat; } Eigen::Ref<Eigen::MatrixXd> ref() { return mat; } Eigen::Ref<const Eigen::MatrixXd> refConst() { return mat; } - Eigen::Block<Eigen::MatrixXd> block(int r, int c, int nrow, int ncol) { return mat.block(r, c, nrow, ncol); } - Eigen::Block<const Eigen::MatrixXd> blockConst(int r, int c, int nrow, int ncol) const { return mat.block(r, c, nrow, ncol); } - py::EigenDMap<Eigen::Matrix2d> corners() { return py::EigenDMap<Eigen::Matrix2d>(mat.data(), - py::EigenDStride(mat.outerStride() * (mat.outerSize()-1), mat.innerStride() * (mat.innerSize()-1))); } - py::EigenDMap<const Eigen::Matrix2d> cornersConst() const { return py::EigenDMap<const Eigen::Matrix2d>(mat.data(), - py::EigenDStride(mat.outerStride() * (mat.outerSize()-1), mat.innerStride() * (mat.innerSize()-1))); } + Eigen::Block<Eigen::MatrixXd> block(int r, int c, int nrow, int ncol) { + return mat.block(r, c, nrow, ncol); + } + Eigen::Block<const Eigen::MatrixXd> blockConst(int r, int c, int nrow, int ncol) const { + return mat.block(r, c, nrow, ncol); + } + py::EigenDMap<Eigen::Matrix2d> corners() { + return py::EigenDMap<Eigen::Matrix2d>( + mat.data(), + py::EigenDStride(mat.outerStride() * (mat.outerSize() - 1), + mat.innerStride() * (mat.innerSize() - 1))); + } + py::EigenDMap<const Eigen::Matrix2d> cornersConst() const { + return py::EigenDMap<const Eigen::Matrix2d>( + mat.data(), + py::EigenDStride(mat.outerStride() * (mat.outerSize() - 1), + mat.innerStride() * (mat.innerSize() - 1))); + } }; using rvp = py::return_value_policy; py::class_<ReturnTester>(m, "ReturnTester") @@ -199,9 +275,9 @@ TEST_SUBMODULE(eigen, m) { .def("get_ptr", &ReturnTester::getPtr, rvp::reference_internal) .def("view", &ReturnTester::view, rvp::reference_internal) .def("view_ptr", &ReturnTester::view, rvp::reference_internal) - .def("copy_get", &ReturnTester::get) // Default rvp: copy - .def("copy_view", &ReturnTester::view) // " - .def("ref", &ReturnTester::ref) // Default for Ref is to reference + .def("copy_get", &ReturnTester::get) // Default rvp: copy + .def("copy_view", &ReturnTester::view) // " + .def("ref", &ReturnTester::ref) // Default for Ref is to reference .def("ref_const", &ReturnTester::refConst) // Likewise, but const .def("ref_safe", &ReturnTester::ref, rvp::reference_internal) .def("ref_const_safe", &ReturnTester::refConst, rvp::reference_internal) @@ -212,52 +288,55 @@ TEST_SUBMODULE(eigen, m) { .def("block_const", &ReturnTester::blockConst, rvp::reference_internal) .def("copy_block", &ReturnTester::block, rvp::copy) .def("corners", &ReturnTester::corners, rvp::reference_internal) - .def("corners_const", &ReturnTester::cornersConst, rvp::reference_internal) - ; + .def("corners_const", &ReturnTester::cornersConst, rvp::reference_internal); // test_special_matrix_objects // Returns a DiagonalMatrix with diagonal (1,2,3,...) m.def("incr_diag", [](int k) { Eigen::DiagonalMatrix<int, Eigen::Dynamic> m(k); - for (int i = 0; i < k; i++) m.diagonal()[i] = i+1; + for (int i = 0; i < k; i++) { + m.diagonal()[i] = i + 1; + } return m; }); // Returns a SelfAdjointView referencing the lower triangle of m - m.def("symmetric_lower", [](const Eigen::MatrixXi &m) { - return m.selfadjointView<Eigen::Lower>(); - }); + m.def("symmetric_lower", + [](const Eigen::MatrixXi &m) { return m.selfadjointView<Eigen::Lower>(); }); // Returns a SelfAdjointView referencing the lower triangle of m - m.def("symmetric_upper", [](const Eigen::MatrixXi &m) { - return m.selfadjointView<Eigen::Upper>(); - }); + m.def("symmetric_upper", + [](const Eigen::MatrixXi &m) { return m.selfadjointView<Eigen::Upper>(); }); // Test matrix for various functions below. Eigen::MatrixXf mat(5, 6); - mat << 0, 3, 0, 0, 0, 11, - 22, 0, 0, 0, 17, 11, - 7, 5, 0, 1, 0, 11, - 0, 0, 0, 0, 0, 11, - 0, 0, 14, 0, 8, 11; + mat << 0, 3, 0, 0, 0, 11, 22, 0, 0, 0, 17, 11, 7, 5, 0, 1, 0, 11, 0, 0, 0, 0, 0, 11, 0, 0, 14, + 0, 8, 11; // test_fixed, and various other tests m.def("fixed_r", [mat]() -> FixedMatrixR { return FixedMatrixR(mat); }); + // Our Eigen does a hack which respects constness through the numpy writeable flag. + // Therefore, the const return actually affects this type despite being an rvalue. + // NOLINTNEXTLINE(readability-const-return-type) m.def("fixed_r_const", [mat]() -> const FixedMatrixR { return FixedMatrixR(mat); }); m.def("fixed_c", [mat]() -> FixedMatrixC { return FixedMatrixC(mat); }); m.def("fixed_copy_r", [](const FixedMatrixR &m) -> FixedMatrixR { return m; }); m.def("fixed_copy_c", [](const FixedMatrixC &m) -> FixedMatrixC { return m; }); // test_mutator_descriptors - m.def("fixed_mutator_r", [](Eigen::Ref<FixedMatrixR>) {}); - m.def("fixed_mutator_c", [](Eigen::Ref<FixedMatrixC>) {}); - m.def("fixed_mutator_a", [](py::EigenDRef<FixedMatrixC>) {}); + m.def("fixed_mutator_r", [](const Eigen::Ref<FixedMatrixR> &) {}); + m.def("fixed_mutator_c", [](const Eigen::Ref<FixedMatrixC> &) {}); + m.def("fixed_mutator_a", [](const py::EigenDRef<FixedMatrixC> &) {}); // test_dense m.def("dense_r", [mat]() -> DenseMatrixR { return DenseMatrixR(mat); }); m.def("dense_c", [mat]() -> DenseMatrixC { return DenseMatrixC(mat); }); m.def("dense_copy_r", [](const DenseMatrixR &m) -> DenseMatrixR { return m; }); m.def("dense_copy_c", [](const DenseMatrixC &m) -> DenseMatrixC { return m; }); // test_sparse, test_sparse_signature - m.def("sparse_r", [mat]() -> SparseMatrixR { return Eigen::SparseView<Eigen::MatrixXf>(mat); }); //NOLINT(clang-analyzer-core.uninitialized.UndefReturn) - m.def("sparse_c", [mat]() -> SparseMatrixC { return Eigen::SparseView<Eigen::MatrixXf>(mat); }); + m.def("sparse_r", [mat]() -> SparseMatrixR { + // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn) + return Eigen::SparseView<Eigen::MatrixXf>(mat); + }); + m.def("sparse_c", + [mat]() -> SparseMatrixC { return Eigen::SparseView<Eigen::MatrixXf>(mat); }); m.def("sparse_copy_r", [](const SparseMatrixR &m) -> SparseMatrixR { return m; }); m.def("sparse_copy_c", [](const SparseMatrixC &m) -> SparseMatrixC { return m; }); // test_partially_fixed @@ -271,7 +350,8 @@ TEST_SUBMODULE(eigen, m) { m.def("cpp_copy", [](py::handle m) { return m.cast<Eigen::MatrixXd>()(1, 0); }); m.def("cpp_ref_c", [](py::handle m) { return m.cast<Eigen::Ref<Eigen::MatrixXd>>()(1, 0); }); m.def("cpp_ref_r", [](py::handle m) { return m.cast<Eigen::Ref<MatrixXdR>>()(1, 0); }); - m.def("cpp_ref_any", [](py::handle m) { return m.cast<py::EigenDRef<Eigen::MatrixXd>>()(1, 0); }); + m.def("cpp_ref_any", + [](py::handle m) { return m.cast<py::EigenDRef<Eigen::MatrixXd>>()(1, 0); }); // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. @@ -280,33 +360,52 @@ TEST_SUBMODULE(eigen, m) { // that would allow copying (if types or strides don't match) for comparison: m.def("get_elem", &get_elem); // Now this alternative that calls the tells pybind to fail rather than copy: - m.def("get_elem_nocopy", [](Eigen::Ref<const Eigen::MatrixXd> m) -> double { return get_elem(m); }, - py::arg{}.noconvert()); + m.def( + "get_elem_nocopy", + [](const Eigen::Ref<const Eigen::MatrixXd> &m) -> double { return get_elem(m); }, + py::arg{}.noconvert()); // Also test a row-major-only no-copy const ref: - m.def("get_elem_rm_nocopy", [](Eigen::Ref<const Eigen::Matrix<long, -1, -1, Eigen::RowMajor>> &m) -> long { return m(2, 1); }, - py::arg{}.noconvert()); - - // test_issue738 - // Issue #738: 1xN or Nx1 2D matrices were neither accepted nor properly copied with an + m.def( + "get_elem_rm_nocopy", + [](Eigen::Ref<const Eigen::Matrix<long, -1, -1, Eigen::RowMajor>> &m) -> long { + return m(2, 1); + }, + py::arg{}.noconvert()); + + // test_issue738, test_zero_length + // Issue #738: 1×N or N×1 2D matrices were neither accepted nor properly copied with an // incompatible stride value on the length-1 dimension--but that should be allowed (without // requiring a copy!) because the stride value can be safely ignored on a size-1 dimension. - m.def("iss738_f1", &adjust_matrix<const Eigen::Ref<const Eigen::MatrixXd> &>, py::arg{}.noconvert()); - m.def("iss738_f2", &adjust_matrix<const Eigen::Ref<const Eigen::Matrix<double, -1, -1, Eigen::RowMajor>> &>, py::arg{}.noconvert()); + // Similarly, 0×N or N×0 matrices were not accepted--again, these should be allowed since + // they contain no data. This particularly affects numpy ≥ 1.23, which sets the strides to + // 0 if any dimension size is 0. + m.def("iss738_f1", + &adjust_matrix<const Eigen::Ref<const Eigen::MatrixXd> &>, + py::arg{}.noconvert()); + m.def("iss738_f2", + &adjust_matrix<const Eigen::Ref<const Eigen::Matrix<double, -1, -1, Eigen::RowMajor>> &>, + py::arg{}.noconvert()); // test_issue1105 // Issue #1105: when converting from a numpy two-dimensional (Nx1) or (1xN) value into a dense - // eigen Vector or RowVector, the argument would fail to load because the numpy copy would fail: - // numpy won't broadcast a Nx1 into a 1-dimensional vector. - m.def("iss1105_col", [](Eigen::VectorXd) { return true; }); - m.def("iss1105_row", [](Eigen::RowVectorXd) { return true; }); + // eigen Vector or RowVector, the argument would fail to load because the numpy copy would + // fail: numpy won't broadcast a Nx1 into a 1-dimensional vector. + m.def("iss1105_col", [](const Eigen::VectorXd &) { return true; }); + m.def("iss1105_row", [](const Eigen::RowVectorXd &) { return true; }); // test_named_arguments // Make sure named arguments are working properly: - m.def("matrix_multiply", [](const py::EigenDRef<const Eigen::MatrixXd> A, const py::EigenDRef<const Eigen::MatrixXd> B) - -> Eigen::MatrixXd { - if (A.cols() != B.rows()) throw std::domain_error("Nonconformable matrices!"); - return A * B; - }, py::arg("A"), py::arg("B")); + m.def( + "matrix_multiply", + [](const py::EigenDRef<const Eigen::MatrixXd> &A, + const py::EigenDRef<const Eigen::MatrixXd> &B) -> Eigen::MatrixXd { + if (A.cols() != B.rows()) { + throw std::domain_error("Nonconformable matrices!"); + } + return A * B; + }, + py::arg("A"), + py::arg("B")); // test_custom_operator_new py::class_<CustomOperatorNew>(m, "CustomOperatorNew") @@ -318,7 +417,7 @@ TEST_SUBMODULE(eigen, m) { // In case of a failure (the caster's temp array does not live long enough), creating // a new array (np.ones(10)) increases the chances that the temp array will be garbage // collected and/or that its memory will be overridden with different values. - m.def("get_elem_direct", [](Eigen::Ref<const Eigen::VectorXd> v) { + m.def("get_elem_direct", [](const Eigen::Ref<const Eigen::VectorXd> &v) { py::module_::import("numpy").attr("ones")(10); return v(5); }); diff --git a/tests/test_eigen.py b/tests/test_eigen_matrix.py index a131dc15..b2e76740 100644 --- a/tests/test_eigen.py +++ b/tests/test_eigen_matrix.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- import pytest + from pybind11_tests import ConstructorStats np = pytest.importorskip("numpy") -m = pytest.importorskip("pybind11_tests.eigen") +m = pytest.importorskip("pybind11_tests.eigen_matrix") ref = np.array( @@ -200,7 +200,7 @@ def test_negative_stride_from_python(msg): double_threer(): incompatible function arguments. The following argument types are supported: 1. (arg0: numpy.ndarray[numpy.float32[1, 3], flags.writeable]) -> None - Invoked with: """ # noqa: E501 line too long + Invoked with: """ + repr(np.array([5.0, 4.0, 3.0], dtype="float32")) ) @@ -212,22 +212,29 @@ def test_negative_stride_from_python(msg): double_threec(): incompatible function arguments. The following argument types are supported: 1. (arg0: numpy.ndarray[numpy.float32[3, 1], flags.writeable]) -> None - Invoked with: """ # noqa: E501 line too long + Invoked with: """ + repr(np.array([7.0, 4.0, 1.0], dtype="float32")) ) +def test_block_runtime_error_type_caster_eigen_ref_made_a_copy(): + with pytest.raises(RuntimeError) as excinfo: + m.block(ref, 0, 0, 0, 0) + assert str(excinfo.value) == "type_caster for Eigen::Ref made a copy." + + def test_nonunit_stride_to_python(): assert np.all(m.diagonal(ref) == ref.diagonal()) assert np.all(m.diagonal_1(ref) == ref.diagonal(1)) for i in range(-5, 7): - assert np.all( - m.diagonal_n(ref, i) == ref.diagonal(i) - ), "m.diagonal_n({})".format(i) + assert np.all(m.diagonal_n(ref, i) == ref.diagonal(i)), f"m.diagonal_n({i})" - assert np.all(m.block(ref, 2, 1, 3, 3) == ref[2:5, 1:4]) - assert np.all(m.block(ref, 1, 4, 4, 2) == ref[1:, 4:]) - assert np.all(m.block(ref, 1, 4, 3, 2) == ref[1:4, 4:]) + # Must be order="F", otherwise the type_caster will make a copy and + # m.block() will return a dangling reference (heap-use-after-free). + rof = np.asarray(ref, order="F") + assert np.all(m.block(rof, 2, 1, 3, 3) == rof[2:5, 1:4]) + assert np.all(m.block(rof, 1, 4, 4, 2) == rof[1:, 4:]) + assert np.all(m.block(rof, 1, 4, 3, 2) == rof[1:4, 4:]) def test_eigen_ref_to_python(): @@ -236,7 +243,7 @@ def test_eigen_ref_to_python(): mymat = chol(np.array([[1.0, 2, 4], [2, 13, 23], [4, 23, 77]])) assert np.all( mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]]) - ), "cholesky{}".format(i) + ), f"cholesky{i}" def assign_both(a1, a2, r, c, v): @@ -253,111 +260,129 @@ def array_copy_but_one(a, r, c, v): def test_eigen_return_references(): """Tests various ways of returning references and non-referencing copies""" - master = np.ones((10, 10)) + primary = np.ones((10, 10)) a = m.ReturnTester() a_get1 = a.get() - assert not a_get1.flags.owndata and a_get1.flags.writeable - assign_both(a_get1, master, 3, 3, 5) + assert not a_get1.flags.owndata + assert a_get1.flags.writeable + assign_both(a_get1, primary, 3, 3, 5) a_get2 = a.get_ptr() - assert not a_get2.flags.owndata and a_get2.flags.writeable - assign_both(a_get1, master, 2, 3, 6) + assert not a_get2.flags.owndata + assert a_get2.flags.writeable + assign_both(a_get1, primary, 2, 3, 6) a_view1 = a.view() - assert not a_view1.flags.owndata and not a_view1.flags.writeable + assert not a_view1.flags.owndata + assert not a_view1.flags.writeable with pytest.raises(ValueError): a_view1[2, 3] = 4 a_view2 = a.view_ptr() - assert not a_view2.flags.owndata and not a_view2.flags.writeable + assert not a_view2.flags.owndata + assert not a_view2.flags.writeable with pytest.raises(ValueError): a_view2[2, 3] = 4 a_copy1 = a.copy_get() - assert a_copy1.flags.owndata and a_copy1.flags.writeable - np.testing.assert_array_equal(a_copy1, master) + assert a_copy1.flags.owndata + assert a_copy1.flags.writeable + np.testing.assert_array_equal(a_copy1, primary) a_copy1[7, 7] = -44 # Shouldn't affect anything else - c1want = array_copy_but_one(master, 7, 7, -44) + c1want = array_copy_but_one(primary, 7, 7, -44) a_copy2 = a.copy_view() - assert a_copy2.flags.owndata and a_copy2.flags.writeable - np.testing.assert_array_equal(a_copy2, master) + assert a_copy2.flags.owndata + assert a_copy2.flags.writeable + np.testing.assert_array_equal(a_copy2, primary) a_copy2[4, 4] = -22 # Shouldn't affect anything else - c2want = array_copy_but_one(master, 4, 4, -22) + c2want = array_copy_but_one(primary, 4, 4, -22) a_ref1 = a.ref() - assert not a_ref1.flags.owndata and a_ref1.flags.writeable - assign_both(a_ref1, master, 1, 1, 15) + assert not a_ref1.flags.owndata + assert a_ref1.flags.writeable + assign_both(a_ref1, primary, 1, 1, 15) a_ref2 = a.ref_const() - assert not a_ref2.flags.owndata and not a_ref2.flags.writeable + assert not a_ref2.flags.owndata + assert not a_ref2.flags.writeable with pytest.raises(ValueError): a_ref2[5, 5] = 33 a_ref3 = a.ref_safe() - assert not a_ref3.flags.owndata and a_ref3.flags.writeable - assign_both(a_ref3, master, 0, 7, 99) + assert not a_ref3.flags.owndata + assert a_ref3.flags.writeable + assign_both(a_ref3, primary, 0, 7, 99) a_ref4 = a.ref_const_safe() - assert not a_ref4.flags.owndata and not a_ref4.flags.writeable + assert not a_ref4.flags.owndata + assert not a_ref4.flags.writeable with pytest.raises(ValueError): a_ref4[7, 0] = 987654321 a_copy3 = a.copy_ref() - assert a_copy3.flags.owndata and a_copy3.flags.writeable - np.testing.assert_array_equal(a_copy3, master) + assert a_copy3.flags.owndata + assert a_copy3.flags.writeable + np.testing.assert_array_equal(a_copy3, primary) a_copy3[8, 1] = 11 - c3want = array_copy_but_one(master, 8, 1, 11) + c3want = array_copy_but_one(primary, 8, 1, 11) a_copy4 = a.copy_ref_const() - assert a_copy4.flags.owndata and a_copy4.flags.writeable - np.testing.assert_array_equal(a_copy4, master) + assert a_copy4.flags.owndata + assert a_copy4.flags.writeable + np.testing.assert_array_equal(a_copy4, primary) a_copy4[8, 4] = 88 - c4want = array_copy_but_one(master, 8, 4, 88) + c4want = array_copy_but_one(primary, 8, 4, 88) a_block1 = a.block(3, 3, 2, 2) - assert not a_block1.flags.owndata and a_block1.flags.writeable + assert not a_block1.flags.owndata + assert a_block1.flags.writeable a_block1[0, 0] = 55 - master[3, 3] = 55 + primary[3, 3] = 55 a_block2 = a.block_safe(2, 2, 3, 2) - assert not a_block2.flags.owndata and a_block2.flags.writeable + assert not a_block2.flags.owndata + assert a_block2.flags.writeable a_block2[2, 1] = -123 - master[4, 3] = -123 + primary[4, 3] = -123 a_block3 = a.block_const(6, 7, 4, 3) - assert not a_block3.flags.owndata and not a_block3.flags.writeable + assert not a_block3.flags.owndata + assert not a_block3.flags.writeable with pytest.raises(ValueError): a_block3[2, 2] = -44444 a_copy5 = a.copy_block(2, 2, 2, 3) - assert a_copy5.flags.owndata and a_copy5.flags.writeable - np.testing.assert_array_equal(a_copy5, master[2:4, 2:5]) + assert a_copy5.flags.owndata + assert a_copy5.flags.writeable + np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5]) a_copy5[1, 1] = 777 - c5want = array_copy_but_one(master[2:4, 2:5], 1, 1, 777) + c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777) a_corn1 = a.corners() - assert not a_corn1.flags.owndata and a_corn1.flags.writeable + assert not a_corn1.flags.owndata + assert a_corn1.flags.writeable a_corn1 *= 50 a_corn1[1, 1] = 999 - master[0, 0] = 50 - master[0, 9] = 50 - master[9, 0] = 50 - master[9, 9] = 999 + primary[0, 0] = 50 + primary[0, 9] = 50 + primary[9, 0] = 50 + primary[9, 9] = 999 a_corn2 = a.corners_const() - assert not a_corn2.flags.owndata and not a_corn2.flags.writeable + assert not a_corn2.flags.owndata + assert not a_corn2.flags.writeable with pytest.raises(ValueError): a_corn2[1, 0] = 51 # All of the changes made all the way along should be visible everywhere # now (except for the copies, of course) - np.testing.assert_array_equal(a_get1, master) - np.testing.assert_array_equal(a_get2, master) - np.testing.assert_array_equal(a_view1, master) - np.testing.assert_array_equal(a_view2, master) - np.testing.assert_array_equal(a_ref1, master) - np.testing.assert_array_equal(a_ref2, master) - np.testing.assert_array_equal(a_ref3, master) - np.testing.assert_array_equal(a_ref4, master) - np.testing.assert_array_equal(a_block1, master[3:5, 3:5]) - np.testing.assert_array_equal(a_block2, master[2:5, 2:4]) - np.testing.assert_array_equal(a_block3, master[6:10, 7:10]) + np.testing.assert_array_equal(a_get1, primary) + np.testing.assert_array_equal(a_get2, primary) + np.testing.assert_array_equal(a_view1, primary) + np.testing.assert_array_equal(a_view2, primary) + np.testing.assert_array_equal(a_ref1, primary) + np.testing.assert_array_equal(a_ref2, primary) + np.testing.assert_array_equal(a_ref3, primary) + np.testing.assert_array_equal(a_ref4, primary) + np.testing.assert_array_equal(a_block1, primary[3:5, 3:5]) + np.testing.assert_array_equal(a_block2, primary[2:5, 2:4]) + np.testing.assert_array_equal(a_block3, primary[6:10, 7:10]) np.testing.assert_array_equal( - a_corn1, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1] + a_corn1, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1] ) np.testing.assert_array_equal( - a_corn2, master[0 :: master.shape[0] - 1, 0 :: master.shape[1] - 1] + a_corn2, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1] ) np.testing.assert_array_equal(a_copy1, c1want) @@ -496,10 +521,14 @@ def test_numpy_ref_mutators(): assert [zc[1, 2], zcro[1, 2], zr[1, 2], zrro[1, 2]] == [23] * 4 - assert not zc.flags.owndata and zc.flags.writeable - assert not zr.flags.owndata and zr.flags.writeable - assert not zcro.flags.owndata and not zcro.flags.writeable - assert not zrro.flags.owndata and not zrro.flags.writeable + assert not zc.flags.owndata + assert zc.flags.writeable + assert not zr.flags.owndata + assert zr.flags.writeable + assert not zcro.flags.owndata + assert not zcro.flags.writeable + assert not zrro.flags.owndata + assert not zrro.flags.writeable zc[1, 2] = 99 expect = np.array([[11.0, 12, 13], [21, 22, 99], [31, 32, 33]]) @@ -523,7 +552,8 @@ def test_numpy_ref_mutators(): # the const should drop away) y1 = np.array(m.get_cm_const_ref()) - assert y1.flags.owndata and y1.flags.writeable + assert y1.flags.owndata + assert y1.flags.writeable # We should get copies of the eigen data, which was modified above: assert y1[1, 2] == 99 y1[1, 2] += 12 @@ -596,38 +626,38 @@ def test_nocopy_wrapper(): # All but the second should fail with m.get_elem_nocopy: with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_colmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(int_matrix_rowmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_nocopy(dbl_matrix_rowmajor) - assert "get_elem_nocopy(): incompatible function arguments." in str( - excinfo.value - ) and ", flags.f_contiguous" in str(excinfo.value) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) # For the row-major test, we take a long matrix in row-major, so only the third is allowed: with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(int_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_colmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8 with pytest.raises(TypeError) as excinfo: m.get_elem_rm_nocopy(dbl_matrix_rowmajor) assert "get_elem_rm_nocopy(): incompatible function arguments." in str( excinfo.value - ) and ", flags.c_contiguous" in str(excinfo.value) + ) + assert ", flags.c_contiguous" in str(excinfo.value) def test_eigen_ref_life_support(): @@ -723,13 +753,13 @@ def test_sparse_signature(doc): doc(m.sparse_copy_r) == """ sparse_copy_r(arg0: scipy.sparse.csr_matrix[numpy.float32]) -> scipy.sparse.csr_matrix[numpy.float32] - """ # noqa: E501 line too long + """ ) assert ( doc(m.sparse_copy_c) == """ sparse_copy_c(arg0: scipy.sparse.csc_matrix[numpy.float32]) -> scipy.sparse.csc_matrix[numpy.float32] - """ # noqa: E501 line too long + """ ) @@ -746,6 +776,13 @@ def test_issue738(): ) +@pytest.mark.parametrize("func", [m.iss738_f1, m.iss738_f2]) +@pytest.mark.parametrize("sizes", [(0, 2), (2, 0)]) +def test_zero_length(func, sizes): + """Ignore strides on a length-0 dimension (even if they would be incompatible length > 1)""" + assert np.all(func(np.zeros(sizes)) == np.zeros(sizes)) + + def test_issue1105(): """Issue 1105: 1xN or Nx1 input arrays weren't accepted for eigen compile-time row vectors or column vector""" diff --git a/tests/test_eigen_tensor.cpp b/tests/test_eigen_tensor.cpp new file mode 100644 index 00000000..503c69c7 --- /dev/null +++ b/tests/test_eigen_tensor.cpp @@ -0,0 +1,18 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor + +#ifdef EIGEN_AVOID_STL_ARRAY +# undef EIGEN_AVOID_STL_ARRAY +#endif + +#include "test_eigen_tensor.inl" + +#include "pybind11_tests.h" + +test_initializer egien_tensor("eigen_tensor", eigen_tensor_test::test_module); diff --git a/tests/test_eigen_tensor.inl b/tests/test_eigen_tensor.inl new file mode 100644 index 00000000..d864ce73 --- /dev/null +++ b/tests/test_eigen_tensor.inl @@ -0,0 +1,333 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include <pybind11/eigen/tensor.h> + +PYBIND11_NAMESPACE_BEGIN(eigen_tensor_test) + +namespace py = pybind11; + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +template <typename M> +void reset_tensor(M &x) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { + x(i, j, k) = i * (5 * 2) + j * 2 + k; + } + } + } +} + +template <typename M> +bool check_tensor(M &x) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { + if (x(i, j, k) != (i * (5 * 2) + j * 2 + k)) { + return false; + } + } + } + } + return true; +} + +template <int Options> +Eigen::Tensor<double, 3, Options> &get_tensor() { + static Eigen::Tensor<double, 3, Options> *x; + + if (!x) { + x = new Eigen::Tensor<double, 3, Options>(3, 5, 2); + reset_tensor(*x); + } + + return *x; +} + +template <int Options> +Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> &get_tensor_map() { + static Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> *x; + + if (!x) { + x = new Eigen::TensorMap<Eigen::Tensor<double, 3, Options>>(get_tensor<Options>()); + } + + return *x; +} + +template <int Options> +Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options> &get_fixed_tensor() { + static Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options> *x; + + if (!x) { + Eigen::aligned_allocator<Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options>> + allocator; + x = new (allocator.allocate(1)) + Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options>(); + reset_tensor(*x); + } + + return *x; +} + +template <int Options> +const Eigen::Tensor<double, 3, Options> &get_const_tensor() { + return get_tensor<Options>(); +} + +template <int Options> +struct CustomExample { + CustomExample() : member(get_tensor<Options>()), view_member(member) {} + + Eigen::Tensor<double, 3, Options> member; + Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> view_member; +}; + +template <int Options> +void init_tensor_module(pybind11::module &m) { + const char *needed_options = ""; + if (Options == Eigen::ColMajor) { + needed_options = "F"; + } else { + needed_options = "C"; + } + m.attr("needed_options") = needed_options; + + m.def("setup", []() { + reset_tensor(get_tensor<Options>()); + reset_tensor(get_fixed_tensor<Options>()); + }); + + m.def("is_ok", []() { + return check_tensor(get_tensor<Options>()) && check_tensor(get_fixed_tensor<Options>()); + }); + + py::class_<CustomExample<Options>>(m, "CustomExample", py::module_local()) + .def(py::init<>()) + .def_readonly( + "member", &CustomExample<Options>::member, py::return_value_policy::reference_internal) + .def_readonly("member_view", + &CustomExample<Options>::view_member, + py::return_value_policy::reference_internal); + + m.def( + "copy_fixed_tensor", + []() { return &get_fixed_tensor<Options>(); }, + py::return_value_policy::copy); + + m.def( + "copy_tensor", []() { return &get_tensor<Options>(); }, py::return_value_policy::copy); + + m.def( + "copy_const_tensor", + []() { return &get_const_tensor<Options>(); }, + py::return_value_policy::copy); + + m.def( + "move_fixed_tensor_copy", + []() -> Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options> { + return get_fixed_tensor<Options>(); + }, + py::return_value_policy::move); + + m.def( + "move_tensor_copy", + []() -> Eigen::Tensor<double, 3, Options> { return get_tensor<Options>(); }, + py::return_value_policy::move); + + m.def( + "move_const_tensor", + []() -> const Eigen::Tensor<double, 3, Options> & { return get_const_tensor<Options>(); }, + py::return_value_policy::move); + + m.def( + "take_fixed_tensor", + + []() { + Eigen::aligned_allocator< + Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options>> + allocator; + return new (allocator.allocate(1)) + Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options>( + get_fixed_tensor<Options>()); + }, + py::return_value_policy::take_ownership); + + m.def( + "take_tensor", + []() { return new Eigen::Tensor<double, 3, Options>(get_tensor<Options>()); }, + py::return_value_policy::take_ownership); + + m.def( + "take_const_tensor", + []() -> const Eigen::Tensor<double, 3, Options> * { + return new Eigen::Tensor<double, 3, Options>(get_tensor<Options>()); + }, + py::return_value_policy::take_ownership); + + m.def( + "take_view_tensor", + []() -> const Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> * { + return new Eigen::TensorMap<Eigen::Tensor<double, 3, Options>>(get_tensor<Options>()); + }, + py::return_value_policy::take_ownership); + + m.def( + "reference_tensor", + []() { return &get_tensor<Options>(); }, + py::return_value_policy::reference); + + m.def( + "reference_tensor_v2", + []() -> Eigen::Tensor<double, 3, Options> & { return get_tensor<Options>(); }, + py::return_value_policy::reference); + + m.def( + "reference_tensor_internal", + []() { return &get_tensor<Options>(); }, + py::return_value_policy::reference_internal); + + m.def( + "reference_fixed_tensor", + []() { return &get_tensor<Options>(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_tensor", + []() { return &get_const_tensor<Options>(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_tensor_v2", + []() -> const Eigen::Tensor<double, 3, Options> & { return get_const_tensor<Options>(); }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor", + []() -> Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> { + return get_tensor_map<Options>(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v2", + // NOLINTNEXTLINE(readability-const-return-type) + []() -> const Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> { + return get_tensor_map<Options>(); // NOLINT(readability-const-return-type) + }, // NOLINT(readability-const-return-type) + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v3", + []() -> Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> * { + return &get_tensor_map<Options>(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v4", + []() -> const Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> * { + return &get_tensor_map<Options>(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v5", + []() -> Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> & { + return get_tensor_map<Options>(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v6", + []() -> const Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> & { + return get_tensor_map<Options>(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_fixed_tensor", + []() { + return Eigen::TensorMap< + Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options>>( + get_fixed_tensor<Options>()); + }, + py::return_value_policy::reference); + + m.def("round_trip_tensor", + [](const Eigen::Tensor<double, 3, Options> &tensor) { return tensor; }); + + m.def( + "round_trip_tensor_noconvert", + [](const Eigen::Tensor<double, 3, Options> &tensor) { return tensor; }, + py::arg("tensor").noconvert()); + + m.def("round_trip_tensor2", + [](const Eigen::Tensor<int32_t, 3, Options> &tensor) { return tensor; }); + + m.def("round_trip_fixed_tensor", + [](const Eigen::TensorFixedSize<double, Eigen::Sizes<3, 5, 2>, Options> &tensor) { + return tensor; + }); + + m.def( + "round_trip_view_tensor", + [](Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ref", + [](Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> &view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ptr", + [](Eigen::TensorMap<Eigen::Tensor<double, 3, Options>> *view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_aligned_view_tensor", + [](Eigen::TensorMap<Eigen::Tensor<double, 3, Options>, Eigen::Aligned> view) { + return view; + }, + py::return_value_policy::reference); + + m.def( + "round_trip_const_view_tensor", + [](Eigen::TensorMap<const Eigen::Tensor<double, 3, Options>> view) { + return Eigen::Tensor<double, 3, Options>(view); + }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0", + [](const Eigen::Tensor<double, 0, Options> &tensor) { return tensor; }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_noconvert", + [](const Eigen::Tensor<double, 0, Options> &tensor) { return tensor; }, + py::arg("tensor").noconvert(), + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_view", + [](Eigen::TensorMap<Eigen::Tensor<double, 0, Options>> &tensor) { return tensor; }, + py::return_value_policy::reference); +} + +void test_module(py::module_ &m) { + auto f_style = m.def_submodule("f_style"); + auto c_style = m.def_submodule("c_style"); + + init_tensor_module<Eigen::ColMajor>(f_style); + init_tensor_module<Eigen::RowMajor>(c_style); +} + +PYBIND11_NAMESPACE_END(eigen_tensor_test) diff --git a/tests/test_eigen_tensor.py b/tests/test_eigen_tensor.py new file mode 100644 index 00000000..3e7ee6b7 --- /dev/null +++ b/tests/test_eigen_tensor.py @@ -0,0 +1,288 @@ +import sys + +import pytest + +np = pytest.importorskip("numpy") +eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") +submodules = [eigen_tensor.c_style, eigen_tensor.f_style] +try: + import eigen_tensor_avoid_stl_array as avoid + + submodules += [avoid.c_style, avoid.f_style] +except ImportError as e: + # Ensure config, build, toolchain, etc. issues are not masked here: + msg = ( + "import eigen_tensor_avoid_stl_array FAILED, while " + "import pybind11_tests.eigen_tensor succeeded. " + "Please ensure that " + "test_eigen_tensor.cpp & " + "eigen_tensor_avoid_stl_array.cpp " + "are built together (or both are not built if Eigen is not available)." + ) + raise RuntimeError(msg) from e + +tensor_ref = np.empty((3, 5, 2), dtype=np.int64) + +for i in range(tensor_ref.shape[0]): + for j in range(tensor_ref.shape[1]): + for k in range(tensor_ref.shape[2]): + tensor_ref[i, j, k] = i * (5 * 2) + j * 2 + k + +indices = (2, 3, 1) + + +@pytest.fixture(autouse=True) +def cleanup(): + for module in submodules: + module.setup() + + yield + + for module in submodules: + assert module.is_ok() + + +def test_import_avoid_stl_array(): + pytest.importorskip("eigen_tensor_avoid_stl_array") + assert len(submodules) == 4 + + +def assert_equal_tensor_ref(mat, writeable=True, modified=None): + assert mat.flags.writeable == writeable + + copy = np.array(tensor_ref) + if modified is not None: + copy[indices] = modified + + np.testing.assert_array_equal(mat, copy) + + +@pytest.mark.parametrize("m", submodules) +@pytest.mark.parametrize("member_name", ["member", "member_view"]) +def test_reference_internal(m, member_name): + if not hasattr(sys, "getrefcount"): + pytest.skip("No reference counting") + foo = m.CustomExample() + counts = sys.getrefcount(foo) + mem = getattr(foo, member_name) + assert_equal_tensor_ref(mem, writeable=False) + new_counts = sys.getrefcount(foo) + assert new_counts == counts + 1 + assert_equal_tensor_ref(mem, writeable=False) + del mem + assert sys.getrefcount(foo) == counts + + +assert_equal_funcs = [ + "copy_tensor", + "copy_fixed_tensor", + "copy_const_tensor", + "move_tensor_copy", + "move_fixed_tensor_copy", + "take_tensor", + "take_fixed_tensor", + "reference_tensor", + "reference_tensor_v2", + "reference_fixed_tensor", + "reference_view_of_tensor", + "reference_view_of_tensor_v3", + "reference_view_of_tensor_v5", + "reference_view_of_fixed_tensor", +] + +assert_equal_const_funcs = [ + "reference_view_of_tensor_v2", + "reference_view_of_tensor_v4", + "reference_view_of_tensor_v6", + "reference_const_tensor", + "reference_const_tensor_v2", +] + + +@pytest.mark.parametrize("m", submodules) +@pytest.mark.parametrize("func_name", assert_equal_funcs + assert_equal_const_funcs) +def test_convert_tensor_to_py(m, func_name): + writeable = func_name in assert_equal_funcs + assert_equal_tensor_ref(getattr(m, func_name)(), writeable=writeable) + + +@pytest.mark.parametrize("m", submodules) +def test_bad_cpp_to_python_casts(m): + with pytest.raises( + RuntimeError, match="Cannot use reference internal when there is no parent" + ): + m.reference_tensor_internal() + + with pytest.raises(RuntimeError, match="Cannot move from a constant reference"): + m.move_const_tensor() + + with pytest.raises( + RuntimeError, match="Cannot take ownership of a const reference" + ): + m.take_const_tensor() + + with pytest.raises( + RuntimeError, + match="Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal", + ): + m.take_view_tensor() + + +@pytest.mark.parametrize("m", submodules) +def test_bad_python_to_cpp_casts(m): + with pytest.raises( + TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments" + ): + m.round_trip_tensor(np.zeros((2, 3))) + + with pytest.raises(TypeError, match=r"^Cannot cast array data from dtype"): + m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) + + with pytest.raises( + TypeError, + match=r"^round_trip_tensor_noconvert\(\): incompatible function arguments", + ): + m.round_trip_tensor_noconvert(tensor_ref) + + assert_equal_tensor_ref( + m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64)) + ) + + bad_options = "C" if m.needed_options == "F" else "F" + # Shape, dtype and the order need to be correct for a TensorMap cast + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + np.zeros((3, 5, 2), dtype=np.float64, order=bad_options) + ) + + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options) + ) + + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + np.zeros((3, 5), dtype=np.float64, order=m.needed_options) + ) + + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + temp[:, ::-1, :], + ) + + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + temp.setflags(write=False) + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor(temp) + + +@pytest.mark.parametrize("m", submodules) +def test_references_actually_refer(m): + a = m.reference_tensor() + temp = a[indices] + a[indices] = 100 + assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) + a[indices] = temp + assert_equal_tensor_ref(m.copy_const_tensor()) + + a = m.reference_view_of_tensor() + a[indices] = 100 + assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) + a[indices] = temp + assert_equal_tensor_ref(m.copy_const_tensor()) + + +@pytest.mark.parametrize("m", submodules) +def test_round_trip(m): + assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + + with pytest.raises(TypeError, match="^Cannot cast array data from"): + assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) + + assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) + assert_equal_tensor_ref(m.round_trip_fixed_tensor(tensor_ref)) + assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) + + copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) + assert_equal_tensor_ref(m.round_trip_view_tensor(copy)) + assert_equal_tensor_ref(m.round_trip_view_tensor_ref(copy)) + assert_equal_tensor_ref(m.round_trip_view_tensor_ptr(copy)) + copy.setflags(write=False) + assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) + + np.testing.assert_array_equal( + tensor_ref[:, ::-1, :], m.round_trip_tensor(tensor_ref[:, ::-1, :]) + ) + + assert m.round_trip_rank_0(np.float64(3.5)) == 3.5 + assert m.round_trip_rank_0(3.5) == 3.5 + + with pytest.raises( + TypeError, + match=r"^round_trip_rank_0_noconvert\(\): incompatible function arguments", + ): + m.round_trip_rank_0_noconvert(np.float64(3.5)) + + with pytest.raises( + TypeError, + match=r"^round_trip_rank_0_noconvert\(\): incompatible function arguments", + ): + m.round_trip_rank_0_noconvert(3.5) + + with pytest.raises( + TypeError, match=r"^round_trip_rank_0_view\(\): incompatible function arguments" + ): + m.round_trip_rank_0_view(np.float64(3.5)) + + with pytest.raises( + TypeError, match=r"^round_trip_rank_0_view\(\): incompatible function arguments" + ): + m.round_trip_rank_0_view(3.5) + + +@pytest.mark.parametrize("m", submodules) +def test_round_trip_references_actually_refer(m): + # Need to create a copy that matches the type on the C side + copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) + a = m.round_trip_view_tensor(copy) + temp = a[indices] + a[indices] = 100 + assert_equal_tensor_ref(copy, modified=100) + a[indices] = temp + assert_equal_tensor_ref(copy) + + +@pytest.mark.parametrize("m", submodules) +def test_doc_string(m, doc): + assert ( + doc(m.copy_tensor) == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + ) + assert ( + doc(m.copy_fixed_tensor) + == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2]]" + ) + assert ( + doc(m.reference_const_tensor) + == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + ) + + order_flag = f"flags.{m.needed_options.lower()}_contiguous" + assert doc(m.round_trip_view_tensor) == ( + f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" + ) + assert doc(m.round_trip_const_view_tensor) == ( + f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])" + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" + ) diff --git a/tests/test_embed/CMakeLists.txt b/tests/test_embed/CMakeLists.txt index c960c877..09a36939 100644 --- a/tests/test_embed/CMakeLists.txt +++ b/tests/test_embed/CMakeLists.txt @@ -7,7 +7,7 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STR return() endif() -find_package(Catch 2.13.2) +find_package(Catch 2.13.9) if(CATCH_FOUND) message(STATUS "Building interpreter tests using Catch v${CATCH_VERSION}") @@ -25,12 +25,13 @@ pybind11_enable_warnings(test_embed) target_link_libraries(test_embed PRIVATE pybind11::embed Catch2::Catch2 Threads::Threads) if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) - file(COPY test_interpreter.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + file(COPY test_interpreter.py test_trampoline.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") endif() add_custom_target( cpptest COMMAND "$<TARGET_FILE:test_embed>" + DEPENDS test_embed WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") pybind11_add_module(external_module THIN_LTO external_module.cpp) diff --git a/tests/test_embed/catch.cpp b/tests/test_embed/catch.cpp index dd137385..558a7a35 100644 --- a/tests/test_embed/catch.cpp +++ b/tests/test_embed/catch.cpp @@ -3,10 +3,13 @@ #include <pybind11/embed.h> -#ifdef _MSC_VER -// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to catch -// 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable: 4996) +// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to +// catch 2.0.1; this should be fixed in the next catch release after 2.0.1). +PYBIND11_WARNING_DISABLE_MSVC(4996) + +// Catch uses _ internally, which breaks gettext style defines +#ifdef _ +# undef _ #endif #define CATCH_CONFIG_RUNNER @@ -15,7 +18,25 @@ namespace py = pybind11; int main(int argc, char *argv[]) { + // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number: + std::string updated_pythonpath("pybind11_test_embed_PYTHONPATH_2099743835476552"); + const char *preexisting_pythonpath = getenv("PYTHONPATH"); + if (preexisting_pythonpath != nullptr) { +#if defined(_WIN32) + updated_pythonpath += ';'; +#else + updated_pythonpath += ':'; +#endif + updated_pythonpath += preexisting_pythonpath; + } +#if defined(_WIN32) + _putenv_s("PYTHONPATH", updated_pythonpath.c_str()); +#else + setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1); +#endif + py::scoped_interpreter guard{}; + auto result = Catch::Session().run(argc, argv); return result < 0xff ? result : 0xff; diff --git a/tests/test_embed/external_module.cpp b/tests/test_embed/external_module.cpp index e9a6058b..5c482fe0 100644 --- a/tests/test_embed/external_module.cpp +++ b/tests/test_embed/external_module.cpp @@ -9,15 +9,12 @@ namespace py = pybind11; PYBIND11_MODULE(external_module, m) { class A { public: - A(int value) : v{value} {}; + explicit A(int value) : v{value} {}; int v; }; - py::class_<A>(m, "A") - .def(py::init<int>()) - .def_readwrite("value", &A::v); + py::class_<A>(m, "A").def(py::init<int>()).def_readwrite("value", &A::v); - m.def("internals_at", []() { - return reinterpret_cast<uintptr_t>(&py::detail::get_internals()); - }); + m.def("internals_at", + []() { return reinterpret_cast<uintptr_t>(&py::detail::get_internals()); }); } diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 944334ce..c6c8a22d 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -1,27 +1,32 @@ #include <pybind11/embed.h> -#ifdef _MSC_VER -// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to catch -// 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable: 4996) -#endif +// Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to +// catch 2.0.1; this should be fixed in the next catch release after 2.0.1). +PYBIND11_WARNING_DISABLE_MSVC(4996) #include <catch.hpp> - -#include <thread> +#include <cstdlib> #include <fstream> #include <functional> +#include <thread> +#include <utility> namespace py = pybind11; using namespace py::literals; +size_t get_sys_path_size() { + auto sys_path = py::module::import("sys").attr("path"); + return py::len(sys_path); +} + class Widget { public: - Widget(std::string message) : message(message) { } + explicit Widget(std::string message) : message(std::move(message)) {} virtual ~Widget() = default; std::string the_message() const { return message; } virtual int the_answer() const = 0; + virtual std::string argv0() const = 0; private: std::string message; @@ -31,6 +36,23 @@ class PyWidget final : public Widget { using Widget::Widget; int the_answer() const override { PYBIND11_OVERRIDE_PURE(int, Widget, the_answer); } + std::string argv0() const override { PYBIND11_OVERRIDE_PURE(std::string, Widget, argv0); } +}; + +class test_override_cache_helper { + +public: + virtual int func() { return 0; } + + test_override_cache_helper() = default; + virtual ~test_override_cache_helper() = default; + // Non-copyable + test_override_cache_helper &operator=(test_override_cache_helper const &Right) = delete; + test_override_cache_helper(test_override_cache_helper const &Copy) = delete; +}; + +class test_override_cache_helper_trampoline : public test_override_cache_helper { + int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); } }; PYBIND11_EMBEDDED_MODULE(widget_module, m) { @@ -41,24 +63,39 @@ PYBIND11_EMBEDDED_MODULE(widget_module, m) { m.def("add", [](int i, int j) { return i + j; }); } -PYBIND11_EMBEDDED_MODULE(throw_exception, ) { - throw std::runtime_error("C++ Error"); +PYBIND11_EMBEDDED_MODULE(trampoline_module, m) { + py::class_<test_override_cache_helper, + test_override_cache_helper_trampoline, + std::shared_ptr<test_override_cache_helper>>(m, "test_override_cache_helper") + .def(py::init_alias<>()) + .def("func", &test_override_cache_helper::func); } +PYBIND11_EMBEDDED_MODULE(throw_exception, ) { throw std::runtime_error("C++ Error"); } + PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) { auto d = py::dict(); d["missing"].cast<py::object>(); } +TEST_CASE("PYTHONPATH is used to update sys.path") { + // The setup for this TEST_CASE is in catch.cpp! + auto sys_path = py::str(py::module_::import("sys").attr("path")).cast<std::string>(); + REQUIRE_THAT(sys_path, + Catch::Matchers::Contains("pybind11_test_embed_PYTHONPATH_2099743835476552")); +} + TEST_CASE("Pass classes and data between modules defined in C++ and Python") { auto module_ = py::module_::import("test_interpreter"); REQUIRE(py::hasattr(module_, "DerivedWidget")); - auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module_.attr("__dict__")); + auto locals = py::dict("hello"_a = "Hello, World!", "x"_a = 5, **module_.attr("__dict__")); py::exec(R"( widget = DerivedWidget("{} - {}".format(hello, x)) message = widget.the_message - )", py::globals(), locals); + )", + py::globals(), + locals); REQUIRE(locals["message"].cast<std::string>() == "Hello, World! - 5"); auto py_widget = module_.attr("DerivedWidget")("The question"); @@ -69,12 +106,51 @@ TEST_CASE("Pass classes and data between modules defined in C++ and Python") { REQUIRE(cpp_widget.the_answer() == 42); } +TEST_CASE("Override cache") { + auto module_ = py::module_::import("test_trampoline"); + REQUIRE(py::hasattr(module_, "func")); + REQUIRE(py::hasattr(module_, "func2")); + + auto locals = py::dict(**module_.attr("__dict__")); + + int i = 0; + for (; i < 1500; ++i) { + std::shared_ptr<test_override_cache_helper> p_obj; + std::shared_ptr<test_override_cache_helper> p_obj2; + + py::object loc_inst = locals["func"](); + p_obj = py::cast<std::shared_ptr<test_override_cache_helper>>(loc_inst); + + int ret = p_obj->func(); + + REQUIRE(ret == 42); + + loc_inst = locals["func2"](); + + p_obj2 = py::cast<std::shared_ptr<test_override_cache_helper>>(loc_inst); + + p_obj2->func(); + } +} + TEST_CASE("Import error handling") { REQUIRE_NOTHROW(py::module_::import("widget_module")); - REQUIRE_THROWS_WITH(py::module_::import("throw_exception"), - "ImportError: C++ Error"); + REQUIRE_THROWS_WITH(py::module_::import("throw_exception"), "ImportError: C++ Error"); REQUIRE_THROWS_WITH(py::module_::import("throw_error_already_set"), - Catch::Contains("ImportError: KeyError")); + Catch::Contains("ImportError: initialization failed")); + + auto locals = py::dict("is_keyerror"_a = false, "message"_a = "not set"); + py::exec(R"( + try: + import throw_error_already_set + except ImportError as e: + is_keyerror = type(e.__cause__) == KeyError + message = str(e.__cause__) + )", + py::globals(), + locals); + REQUIRE(locals["is_keyerror"].cast<bool>() == true); + REQUIRE(locals["message"].cast<std::string>() == "'missing'"); } TEST_CASE("There can be only one interpreter") { @@ -95,26 +171,111 @@ TEST_CASE("There can be only one interpreter") { py::initialize_interpreter(); } -bool has_pybind11_internals_builtin() { - auto builtins = py::handle(PyEval_GetBuiltins()); - return builtins.contains(PYBIND11_INTERNALS_ID); -}; +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Custom PyConfig") { + py::finalize_interpreter(); + PyConfig config; + PyConfig_InitPythonConfig(&config); + REQUIRE_NOTHROW(py::scoped_interpreter{&config}); + { + py::scoped_interpreter p{&config}; + REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast<int>() == 42); + } + py::initialize_interpreter(); +} + +TEST_CASE("scoped_interpreter with PyConfig_InitIsolatedConfig and argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + char *argv[] = {strdup("a.out")}; + py::scoped_interpreter argv_scope{&config, 1, argv}; + std::free(argv[0]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast<const Widget &>(); + REQUIRE(cpp_widget.argv0() == "a.out"); + } + py::initialize_interpreter(); +} + +TEST_CASE("scoped_interpreter with PyConfig_InitPythonConfig and argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + + // `initialize_interpreter() overrides the default value for config.parse_argv (`1`) by + // changing it to `0`. This test exercises `scoped_interpreter` with the default config. + char *argv[] = {strdup("a.out"), strdup("arg1")}; + py::scoped_interpreter argv_scope(&config, 2, argv); + std::free(argv[0]); + std::free(argv[1]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast<const Widget &>(); + REQUIRE(cpp_widget.argv0() == "arg1"); + } + py::initialize_interpreter(); +} +#endif + +TEST_CASE("Add program dir to path pre-PyConfig") { + py::finalize_interpreter(); + size_t path_size_add_program_dir_to_path_false = 0; + { + py::scoped_interpreter scoped_interp{true, 0, nullptr, false}; + path_size_add_program_dir_to_path_false = get_sys_path_size(); + } + { + py::scoped_interpreter scoped_interp{}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); + } + py::initialize_interpreter(); +} + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Add program dir to path using PyConfig") { + py::finalize_interpreter(); + size_t path_size_add_program_dir_to_path_false = 0; + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config, 0, nullptr, false}; + path_size_add_program_dir_to_path_false = get_sys_path_size(); + } + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); + } + py::initialize_interpreter(); +} +#endif + +bool has_state_dict_internals_obj() { + return bool( + py::detail::get_internals_obj_from_state_dict(py::detail::get_python_state_dict())); +} bool has_pybind11_internals_static() { auto **&ipp = py::detail::get_internals_pp(); - return ipp && *ipp; + return (ipp != nullptr) && (*ipp != nullptr); } TEST_CASE("Restart the interpreter") { // Verify pre-restart state. REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast<int>() == 3); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); - REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast<int>() == 123); + REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast<int>() + == 123); // local and foreign module internals should point to the same internals: - REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) == - py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>()); + REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) + == py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>()); // Restart the interpreter. py::finalize_interpreter(); @@ -124,28 +285,31 @@ TEST_CASE("Restart the interpreter") { REQUIRE(Py_IsInitialized() == 1); // Internals are deleted after a restart. - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); pybind11::detail::get_internals(); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); - REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) == - py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>()); + REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp()) + == py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>()); // Make sure that an interpreter with no get_internals() created until finalize still gets the // internals destroyed py::finalize_interpreter(); py::initialize_interpreter(); bool ran = false; - py::module_::import("__main__").attr("internals_destroy_test") = - py::capsule(&ran, [](void *ran) { py::detail::get_internals(); *static_cast<bool *>(ran) = true; }); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + py::module_::import("__main__").attr("internals_destroy_test") + = py::capsule(&ran, [](void *ran) { + py::detail::get_internals(); + *static_cast<bool *>(ran) = true; + }); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); REQUIRE_FALSE(ran); py::finalize_interpreter(); REQUIRE(ran); py::initialize_interpreter(); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); // C++ modules can be reloaded. @@ -167,17 +331,17 @@ TEST_CASE("Subinterpreter") { REQUIRE(m.attr("add")(1, 2).cast<int>() == 3); } - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); /// Create and switch to a subinterpreter. - auto main_tstate = PyThreadState_Get(); - auto sub_tstate = Py_NewInterpreter(); + auto *main_tstate = PyThreadState_Get(); + auto *sub_tstate = Py_NewInterpreter(); // Subinterpreters get their own copy of builtins. detail::get_internals() still // works by returning from the static variable, i.e. all interpreters share a single // global pybind11::internals; - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); // Modules tags should be gone. @@ -212,11 +376,10 @@ TEST_CASE("Threads") { REQUIRE_FALSE(has_pybind11_internals_static()); constexpr auto num_threads = 10; - auto locals = py::dict("count"_a=0); + auto locals = py::dict("count"_a = 0); { py::gil_scoped_release gil_release{}; - REQUIRE(has_pybind11_internals_static()); auto threads = std::vector<std::thread>(); for (auto i = 0; i < num_threads; ++i) { @@ -238,7 +401,11 @@ TEST_CASE("Threads") { struct scope_exit { std::function<void()> f_; explicit scope_exit(std::function<void()> f) noexcept : f_(std::move(f)) {} - ~scope_exit() { if (f_) f_(); } + ~scope_exit() { + if (f_) { + f_(); + } + } }; TEST_CASE("Reload module from file") { @@ -249,9 +416,8 @@ TEST_CASE("Reload module from file") { bool dont_write_bytecode = sys.attr("dont_write_bytecode").cast<bool>(); sys.attr("dont_write_bytecode") = true; // Reset the value at scope exit - scope_exit reset_dont_write_bytecode([&]() { - sys.attr("dont_write_bytecode") = dont_write_bytecode; - }); + scope_exit reset_dont_write_bytecode( + [&]() { sys.attr("dont_write_bytecode") = dont_write_bytecode; }); std::string module_name = "test_module_reload"; std::string module_file = module_name + ".py"; @@ -262,9 +428,7 @@ TEST_CASE("Reload module from file") { test_module << " return 1\n"; test_module.close(); // Delete the file at scope exit - scope_exit delete_module_file([&]() { - std::remove(module_file.c_str()); - }); + scope_exit delete_module_file([&]() { std::remove(module_file.c_str()); }); // Import the module from file auto module_ = py::module_::import(module_name.c_str()); @@ -282,3 +446,43 @@ TEST_CASE("Reload module from file") { result = module_.attr("test")().cast<int>(); REQUIRE(result == 2); } + +TEST_CASE("sys.argv gets initialized properly") { + py::finalize_interpreter(); + { + py::scoped_interpreter default_scope; + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast<const Widget &>(); + REQUIRE(cpp_widget.argv0().empty()); + } + + { + char *argv[] = {strdup("a.out")}; + py::scoped_interpreter argv_scope(true, 1, argv); + std::free(argv[0]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast<const Widget &>(); + REQUIRE(cpp_widget.argv0() == "a.out"); + } + py::initialize_interpreter(); +} + +TEST_CASE("make_iterator can be called before then after finalizing an interpreter") { + // Reproduction of issue #2101 (https://github.com/pybind/pybind11/issues/2101) + py::finalize_interpreter(); + + std::vector<int> container; + { + pybind11::scoped_interpreter g; + auto iter = pybind11::make_iterator(container.begin(), container.end()); + } + + REQUIRE_NOTHROW([&]() { + pybind11::scoped_interpreter g; + auto iter = pybind11::make_iterator(container.begin(), container.end()); + }()); + + py::initialize_interpreter(); +} diff --git a/tests/test_embed/test_interpreter.py b/tests/test_embed/test_interpreter.py index 6174ede4..f2794497 100644 --- a/tests/test_embed/test_interpreter.py +++ b/tests/test_embed/test_interpreter.py @@ -1,10 +1,14 @@ -# -*- coding: utf-8 -*- +import sys + from widget_module import Widget class DerivedWidget(Widget): def __init__(self, message): - super(DerivedWidget, self).__init__(message) + super().__init__(message) def the_answer(self): return 42 + + def argv0(self): + return sys.argv[0] diff --git a/tests/test_embed/test_trampoline.py b/tests/test_embed/test_trampoline.py new file mode 100644 index 00000000..8e14e8ef --- /dev/null +++ b/tests/test_embed/test_trampoline.py @@ -0,0 +1,16 @@ +import trampoline_module + + +def func(): + class Test(trampoline_module.test_override_cache_helper): + def func(self): + return 42 + + return Test() + + +def func2(): + class Test(trampoline_module.test_override_cache_helper): + pass + + return Test() diff --git a/tests/test_enum.cpp b/tests/test_enum.cpp index 31530892..2597b275 100644 --- a/tests/test_enum.cpp +++ b/tests/test_enum.cpp @@ -11,11 +11,7 @@ TEST_SUBMODULE(enums, m) { // test_unscoped_enum - enum UnscopedEnum { - EOne = 1, - ETwo, - EThree - }; + enum UnscopedEnum { EOne = 1, ETwo, EThree }; py::enum_<UnscopedEnum>(m, "UnscopedEnum", py::arithmetic(), "An unscoped enumeration") .value("EOne", EOne, "Docstring for EOne") .value("ETwo", ETwo, "Docstring for ETwo") @@ -23,10 +19,7 @@ TEST_SUBMODULE(enums, m) { .export_values(); // test_scoped_enum - enum class ScopedEnum { - Two = 2, - Three - }; + enum class ScopedEnum { Two = 2, Three }; py::enum_<ScopedEnum>(m, "ScopedEnum", py::arithmetic()) .value("Two", ScopedEnum::Two) .value("Three", ScopedEnum::Three); @@ -36,11 +29,7 @@ TEST_SUBMODULE(enums, m) { }); // test_binary_operators - enum Flags { - Read = 4, - Write = 2, - Execute = 1 - }; + enum Flags { Read = 4, Write = 2, Execute = 1 }; py::enum_<Flags>(m, "Flags", py::arithmetic()) .value("Read", Flags::Read) .value("Write", Flags::Write) @@ -50,14 +39,9 @@ TEST_SUBMODULE(enums, m) { // test_implicit_conversion class ClassWithUnscopedEnum { public: - enum EMode { - EFirstMode = 1, - ESecondMode - }; - - static EMode test_function(EMode mode) { - return mode; - } + enum EMode { EFirstMode = 1, ESecondMode }; + + static EMode test_function(EMode mode) { return mode; } }; py::class_<ClassWithUnscopedEnum> exenum_class(m, "ClassWithUnscopedEnum"); exenum_class.def_static("test_function", &ClassWithUnscopedEnum::test_function); @@ -67,21 +51,83 @@ TEST_SUBMODULE(enums, m) { .export_values(); // test_enum_to_int - m.def("test_enum_to_int", [](int) { }); - m.def("test_enum_to_uint", [](uint32_t) { }); - m.def("test_enum_to_long_long", [](long long) { }); + m.def("test_enum_to_int", [](int) {}); + m.def("test_enum_to_uint", [](uint32_t) {}); + m.def("test_enum_to_long_long", [](long long) {}); // test_duplicate_enum_name - enum SimpleEnum - { - ONE, TWO, THREE - }; + enum SimpleEnum { ONE, TWO, THREE }; m.def("register_bad_enum", [m]() { py::enum_<SimpleEnum>(m, "SimpleEnum") - .value("ONE", SimpleEnum::ONE) //NOTE: all value function calls are called with the same first parameter value + .value("ONE", SimpleEnum::ONE) // NOTE: all value function calls are called with the + // same first parameter value .value("ONE", SimpleEnum::TWO) .value("ONE", SimpleEnum::THREE) .export_values(); }); + + // test_enum_scalar + enum UnscopedUCharEnum : unsigned char {}; + enum class ScopedShortEnum : short {}; + enum class ScopedLongEnum : long {}; + enum UnscopedUInt64Enum : std::uint64_t {}; + static_assert( + py::detail::all_of< + std::is_same<py::enum_<UnscopedUCharEnum>::Scalar, unsigned char>, + std::is_same<py::enum_<ScopedShortEnum>::Scalar, short>, + std::is_same<py::enum_<ScopedLongEnum>::Scalar, long>, + std::is_same<py::enum_<UnscopedUInt64Enum>::Scalar, std::uint64_t>>::value, + "Error during the deduction of enum's scalar type with normal integer underlying"); + + // test_enum_scalar_with_char_underlying + enum class ScopedCharEnum : char { Zero, Positive }; + enum class ScopedWCharEnum : wchar_t { Zero, Positive }; + enum class ScopedChar32Enum : char32_t { Zero, Positive }; + enum class ScopedChar16Enum : char16_t { Zero, Positive }; + + // test the scalar of char type enums according to chapter 'Character types' + // from https://en.cppreference.com/w/cpp/language/types + static_assert( + py::detail::any_of< + std::is_same<py::enum_<ScopedCharEnum>::Scalar, signed char>, // e.g. gcc on x86 + std::is_same<py::enum_<ScopedCharEnum>::Scalar, unsigned char> // e.g. arm linux + >::value, + "char should be cast to either signed char or unsigned char"); + static_assert(sizeof(py::enum_<ScopedWCharEnum>::Scalar) == 2 + || sizeof(py::enum_<ScopedWCharEnum>::Scalar) == 4, + "wchar_t should be either 16 bits (Windows) or 32 (everywhere else)"); + static_assert( + py::detail::all_of< + std::is_same<py::enum_<ScopedChar32Enum>::Scalar, std::uint_least32_t>, + std::is_same<py::enum_<ScopedChar16Enum>::Scalar, std::uint_least16_t>>::value, + "char32_t, char16_t (and char8_t)'s size, signedness, and alignment is determined"); +#if defined(PYBIND11_HAS_U8STRING) + enum class ScopedChar8Enum : char8_t { Zero, Positive }; + static_assert(std::is_same<py::enum_<ScopedChar8Enum>::Scalar, unsigned char>::value); +#endif + + // test_char_underlying_enum + py::enum_<ScopedCharEnum>(m, "ScopedCharEnum") + .value("Zero", ScopedCharEnum::Zero) + .value("Positive", ScopedCharEnum::Positive); + py::enum_<ScopedWCharEnum>(m, "ScopedWCharEnum") + .value("Zero", ScopedWCharEnum::Zero) + .value("Positive", ScopedWCharEnum::Positive); + py::enum_<ScopedChar32Enum>(m, "ScopedChar32Enum") + .value("Zero", ScopedChar32Enum::Zero) + .value("Positive", ScopedChar32Enum::Positive); + py::enum_<ScopedChar16Enum>(m, "ScopedChar16Enum") + .value("Zero", ScopedChar16Enum::Zero) + .value("Positive", ScopedChar16Enum::Positive); + + // test_bool_underlying_enum + enum class ScopedBoolEnum : bool { FALSE, TRUE }; + + // bool is unsigned (std::is_signed returns false) and 1-byte long, so represented with u8 + static_assert(std::is_same<py::enum_<ScopedBoolEnum>::Scalar, std::uint8_t>::value, ""); + + py::enum_<ScopedBoolEnum>(m, "ScopedBoolEnum") + .value("FALSE", ScopedBoolEnum::FALSE) + .value("TRUE", ScopedBoolEnum::TRUE); } diff --git a/tests/test_enum.py b/tests/test_enum.py index e9732fa7..4e85d29c 100644 --- a/tests/test_enum.py +++ b/tests/test_enum.py @@ -1,5 +1,7 @@ -# -*- coding: utf-8 -*- +# ruff: noqa: SIM201 SIM300 SIM202 + import pytest + from pybind11_tests import enums as m @@ -94,13 +96,13 @@ Members: y >= object() # noqa: B015 with pytest.raises(TypeError): - y | object() # noqa: B015 + y | object() with pytest.raises(TypeError): - y & object() # noqa: B015 + y & object() with pytest.raises(TypeError): - y ^ object() # noqa: B015 + y ^ object() assert int(m.UnscopedEnum.ETwo) == 2 assert str(m.UnscopedEnum(2)) == "UnscopedEnum.ETwo" @@ -217,10 +219,16 @@ def test_binary_operators(): def test_enum_to_int(): m.test_enum_to_int(m.Flags.Read) m.test_enum_to_int(m.ClassWithUnscopedEnum.EMode.EFirstMode) + m.test_enum_to_int(m.ScopedCharEnum.Positive) + m.test_enum_to_int(m.ScopedBoolEnum.TRUE) m.test_enum_to_uint(m.Flags.Read) m.test_enum_to_uint(m.ClassWithUnscopedEnum.EMode.EFirstMode) + m.test_enum_to_uint(m.ScopedCharEnum.Positive) + m.test_enum_to_uint(m.ScopedBoolEnum.TRUE) m.test_enum_to_long_long(m.Flags.Read) m.test_enum_to_long_long(m.ClassWithUnscopedEnum.EMode.EFirstMode) + m.test_enum_to_long_long(m.ScopedCharEnum.Positive) + m.test_enum_to_long_long(m.ScopedBoolEnum.TRUE) def test_duplicate_enum_name(): @@ -229,6 +237,28 @@ def test_duplicate_enum_name(): assert str(excinfo.value) == 'SimpleEnum: element "ONE" already exists!' +def test_char_underlying_enum(): # Issue #1331/PR #1334: + assert type(m.ScopedCharEnum.Positive.__int__()) is int + assert int(m.ScopedChar16Enum.Zero) == 0 + assert hash(m.ScopedChar32Enum.Positive) == 1 + assert type(m.ScopedCharEnum.Positive.__getstate__()) is int + assert m.ScopedWCharEnum(1) == m.ScopedWCharEnum.Positive + with pytest.raises(TypeError): + # Even if the underlying type is char, only an int can be used to construct the enum: + m.ScopedCharEnum("0") + + +def test_bool_underlying_enum(): + assert type(m.ScopedBoolEnum.TRUE.__int__()) is int + assert int(m.ScopedBoolEnum.FALSE) == 0 + assert hash(m.ScopedBoolEnum.TRUE) == 1 + assert type(m.ScopedBoolEnum.TRUE.__getstate__()) is int + assert m.ScopedBoolEnum(1) == m.ScopedBoolEnum.TRUE + # Enum could construct with a bool + # (bool is a strict subclass of int, and False will be converted to 0) + assert m.ScopedBoolEnum(False) == m.ScopedBoolEnum.FALSE + + def test_docstring_signatures(): for enum_type in [m.ScopedEnum, m.UnscopedEnum]: for attr in enum_type.__dict__.values(): diff --git a/tests/test_eval.cpp b/tests/test_eval.cpp index 5416c2ec..cd2903f0 100644 --- a/tests/test_eval.cpp +++ b/tests/test_eval.cpp @@ -7,10 +7,12 @@ BSD-style license that can be found in the LICENSE file. */ - #include <pybind11/eval.h> + #include "pybind11_tests.h" +#include <utility> + TEST_SUBMODULE(eval_, m) { // test_evals @@ -18,16 +20,13 @@ TEST_SUBMODULE(eval_, m) { m.def("test_eval_statements", [global]() { auto local = py::dict(); - local["call_test"] = py::cpp_function([&]() -> int { - return 42; - }); + local["call_test"] = py::cpp_function([&]() -> int { return 42; }); // Regular string literal - py::exec( - "message = 'Hello World!'\n" - "x = call_test()", - global, local - ); + py::exec("message = 'Hello World!'\n" + "x = call_test()", + global, + local); // Multi-line raw string literal py::exec(R"( @@ -35,8 +34,9 @@ TEST_SUBMODULE(eval_, m) { print(message) else: raise RuntimeError - )", global, local - ); + )", + global, + local); auto x = local["x"].cast<int>(); return x == 42; @@ -51,9 +51,7 @@ TEST_SUBMODULE(eval_, m) { m.def("test_eval_single_statement", []() { auto local = py::dict(); - local["call_test"] = py::cpp_function([&]() -> int { - return 42; - }); + local["call_test"] = py::cpp_function([&]() -> int { return 42; }); auto result = py::eval<py::eval_single_statement>("x = call_test()", py::dict(), local); auto x = local["x"].cast<int>(); @@ -64,10 +62,10 @@ TEST_SUBMODULE(eval_, m) { auto local = py::dict(); local["y"] = py::int_(43); - int val_out; + int val_out = 0; local["call_test2"] = py::cpp_function([&](int value) { val_out = value; }); - auto result = py::eval_file(filename, global, local); + auto result = py::eval_file(std::move(filename), global, local); return val_out == 43 && result.is_none(); }); @@ -91,9 +89,30 @@ TEST_SUBMODULE(eval_, m) { // test_eval_empty_globals m.def("eval_empty_globals", [](py::object global) { - if (global.is_none()) + if (global.is_none()) { global = py::dict(); + } auto int_class = py::eval("isinstance(42, int)", global); return global; }); + + // test_eval_closure + m.def("test_eval_closure", []() { + py::dict global; + global["closure_value"] = 42; + py::dict local; + local["closure_value"] = 0; + py::exec(R"( + local_value = closure_value + + def func_global(): + return closure_value + + def func_local(): + return local_value + )", + global, + local); + return std::make_pair(global, local); + }); } diff --git a/tests/test_eval.py b/tests/test_eval.py index 1bb05af0..51b6b796 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- import os import pytest import env # noqa: F401 - from pybind11_tests import eval_ as m @@ -19,7 +17,7 @@ def test_evals(capture): assert m.test_eval_failure() -@pytest.mark.xfail("env.PYPY and not env.PY2", raises=RuntimeError) +@pytest.mark.xfail("env.PYPY", raises=RuntimeError) def test_eval_file(): filename = os.path.join(os.path.dirname(__file__), "test_eval_call.py") assert m.test_eval_file(filename) @@ -33,3 +31,20 @@ def test_eval_empty_globals(): g = {} assert "__builtins__" in m.eval_empty_globals(g) assert "__builtins__" in g + + +def test_eval_closure(): + global_, local = m.test_eval_closure() + + assert global_["closure_value"] == 42 + assert local["closure_value"] == 0 + + assert "local_value" not in global_ + assert local["local_value"] == 0 + + assert "func_global" not in global_ + assert local["func_global"]() == 42 + + assert "func_local" not in global_ + with pytest.raises(NameError): + local["func_local"]() diff --git a/tests/test_eval_call.py b/tests/test_eval_call.py index 373b67ba..fd1da2a5 100644 --- a/tests/test_eval_call.py +++ b/tests/test_eval_call.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # This file is called from 'test_eval.py' if "call_test2" in locals(): diff --git a/tests/test_exceptions.cpp b/tests/test_exceptions.cpp index e27c16df..854c7e6f 100644 --- a/tests/test_exceptions.cpp +++ b/tests/test_exceptions.cpp @@ -6,14 +6,21 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#include "test_exceptions.h" +#include "local_bindings.h" #include "pybind11_tests.h" +#include <exception> +#include <stdexcept> +#include <utility> + // A type that should be raised as an exception in Python class MyException : public std::exception { public: - explicit MyException(const char * m) : message{m} {} - const char * what() const noexcept override {return message.c_str();} + explicit MyException(const char *m) : message{m} {} + const char *what() const noexcept override { return message.c_str(); } + private: std::string message = ""; }; @@ -21,8 +28,9 @@ private: // A type that should be translated to a standard Python exception class MyException2 : public std::exception { public: - explicit MyException2(const char * m) : message{m} {} - const char * what() const noexcept override {return message.c_str();} + explicit MyException2(const char *m) : message{m} {} + const char *what() const noexcept override { return message.c_str(); } + private: std::string message = ""; }; @@ -30,13 +38,13 @@ private: // A type that is not derived from std::exception (and is thus unknown) class MyException3 { public: - explicit MyException3(const char * m) : message{m} {} - virtual const char * what() const noexcept {return message.c_str();} + explicit MyException3(const char *m) : message{m} {} + virtual const char *what() const noexcept { return message.c_str(); } // Rule of 5 BEGIN: to preempt compiler warnings. - MyException3(const MyException3&) = default; - MyException3(MyException3&&) = default; - MyException3& operator=(const MyException3&) = default; - MyException3& operator=(MyException3&&) = default; + MyException3(const MyException3 &) = default; + MyException3(MyException3 &&) = default; + MyException3 &operator=(const MyException3 &) = default; + MyException3 &operator=(MyException3 &&) = default; virtual ~MyException3() = default; // Rule of 5 END. private: @@ -47,13 +55,13 @@ private: // and delegated to its exception translator class MyException4 : public std::exception { public: - explicit MyException4(const char * m) : message{m} {} - const char * what() const noexcept override {return message.c_str();} + explicit MyException4(const char *m) : message{m} {} + const char *what() const noexcept override { return message.c_str(); } + private: std::string message = ""; }; - // Like the above, but declared via the helper function class MyException5 : public std::logic_error { public: @@ -65,24 +73,31 @@ class MyException5_1 : public MyException5 { using MyException5::MyException5; }; +// Exception that will be caught via the module local translator. +class MyException6 : public std::exception { +public: + explicit MyException6(const char *m) : message{m} {} + const char *what() const noexcept override { return message.c_str(); } + +private: + std::string message = ""; +}; + struct PythonCallInDestructor { - PythonCallInDestructor(const py::dict &d) : d(d) {} + explicit PythonCallInDestructor(const py::dict &d) : d(d) {} ~PythonCallInDestructor() { d["good"] = true; } py::dict d; }; - - struct PythonAlreadySetInDestructor { - PythonAlreadySetInDestructor(const py::str &s) : s(s) {} + explicit PythonAlreadySetInDestructor(const py::str &s) : s(s) {} ~PythonAlreadySetInDestructor() { py::dict foo; try { // Assign to a py::object to force read access of nonexistent dict entry py::object o = foo["bar"]; - } - catch (py::error_already_set& ex) { + } catch (py::error_already_set &ex) { ex.discard_as_unraisable(s); } } @@ -90,17 +105,17 @@ struct PythonAlreadySetInDestructor { py::str s; }; - TEST_SUBMODULE(exceptions, m) { - m.def("throw_std_exception", []() { - throw std::runtime_error("This exception was intentionally thrown."); - }); + m.def("throw_std_exception", + []() { throw std::runtime_error("This exception was intentionally thrown."); }); // make a new custom exception and use it as a translation target static py::exception<MyException> ex(m, "MyException"); py::register_exception_translator([](std::exception_ptr p) { try { - if (p) std::rethrow_exception(p); + if (p) { + std::rethrow_exception(p); + } } catch (const MyException &e) { // Set MyException as the active python error ex(e.what()); @@ -112,7 +127,9 @@ TEST_SUBMODULE(exceptions, m) { // never by visible from Python py::register_exception_translator([](std::exception_ptr p) { try { - if (p) std::rethrow_exception(p); + if (p) { + std::rethrow_exception(p); + } } catch (const MyException2 &e) { // Translate this exception to a standard RuntimeError PyErr_SetString(PyExc_RuntimeError, e.what()); @@ -124,7 +141,9 @@ TEST_SUBMODULE(exceptions, m) { // translator for MyException by throwing a new exception py::register_exception_translator([](std::exception_ptr p) { try { - if (p) std::rethrow_exception(p); + if (p) { + std::rethrow_exception(p); + } } catch (const MyException4 &e) { throw MyException(e.what()); } @@ -135,22 +154,42 @@ TEST_SUBMODULE(exceptions, m) { // A slightly more complicated one that declares MyException5_1 as a subclass of MyException5 py::register_exception<MyException5_1>(m, "MyException5_1", ex5.ptr()); + // py::register_local_exception<LocalSimpleException>(m, "LocalSimpleException") + + py::register_local_exception_translator([](std::exception_ptr p) { + try { + if (p) { + std::rethrow_exception(p); + } + } catch (const MyException6 &e) { + PyErr_SetString(PyExc_RuntimeError, e.what()); + } + }); + m.def("throws1", []() { throw MyException("this error should go to a custom type"); }); - m.def("throws2", []() { throw MyException2("this error should go to a standard Python exception"); }); + m.def("throws2", + []() { throw MyException2("this error should go to a standard Python exception"); }); m.def("throws3", []() { throw MyException3("this error cannot be translated"); }); m.def("throws4", []() { throw MyException4("this error is rethrown"); }); - m.def("throws5", []() { throw MyException5("this is a helper-defined translated exception"); }); + m.def("throws5", + []() { throw MyException5("this is a helper-defined translated exception"); }); m.def("throws5_1", []() { throw MyException5_1("MyException5 subclass"); }); - m.def("throws_logic_error", []() { throw std::logic_error("this error should fall through to the standard handler"); }); - m.def("throws_overflow_error", []() {throw std::overflow_error(""); }); + m.def("throws6", []() { throw MyException6("MyException6 only handled in this module"); }); + m.def("throws_logic_error", []() { + throw std::logic_error("this error should fall through to the standard handler"); + }); + m.def("throws_overflow_error", []() { throw std::overflow_error(""); }); + m.def("throws_local_error", []() { throw LocalException("never caught"); }); + m.def("throws_local_simple_error", []() { throw LocalSimpleException("this mod"); }); m.def("exception_matches", []() { py::dict foo; try { // Assign to a py::object to force read access of nonexistent dict entry py::object o = foo["bar"]; - } - catch (py::error_already_set& ex) { - if (!ex.matches(PyExc_KeyError)) throw; + } catch (py::error_already_set &ex) { + if (!ex.matches(PyExc_KeyError)) { + throw; + } return true; } return false; @@ -160,9 +199,10 @@ TEST_SUBMODULE(exceptions, m) { try { // Assign to a py::object to force read access of nonexistent dict entry py::object o = foo["bar"]; - } - catch (py::error_already_set &ex) { - if (!ex.matches(PyExc_Exception)) throw; + } catch (py::error_already_set &ex) { + if (!ex.matches(PyExc_Exception)) { + throw; + } return true; } return false; @@ -171,61 +211,137 @@ TEST_SUBMODULE(exceptions, m) { try { // On Python >= 3.6, this raises a ModuleNotFoundError, a subclass of ImportError py::module_::import("nonexistent"); - } - catch (py::error_already_set &ex) { - if (!ex.matches(PyExc_ImportError)) throw; + } catch (py::error_already_set &ex) { + if (!ex.matches(PyExc_ImportError)) { + throw; + } return true; } return false; }); m.def("throw_already_set", [](bool err) { - if (err) + if (err) { PyErr_SetString(PyExc_ValueError, "foo"); + } try { throw py::error_already_set(); - } catch (const std::runtime_error& e) { - if ((err && e.what() != std::string("ValueError: foo")) || - (!err && e.what() != std::string("Unknown internal error occurred"))) - { + } catch (const std::runtime_error &e) { + if ((err && e.what() != std::string("ValueError: foo")) + || (!err + && e.what() + != std::string("Internal error: pybind11::error_already_set called " + "while Python error indicator not set."))) { PyErr_Clear(); throw std::runtime_error("error message mismatch"); } } PyErr_Clear(); - if (err) + if (err) { PyErr_SetString(PyExc_ValueError, "foo"); + } throw py::error_already_set(); }); - m.def("python_call_in_destructor", [](py::dict d) { + m.def("python_call_in_destructor", [](const py::dict &d) { + bool retval = false; try { PythonCallInDestructor set_dict_in_destructor(d); PyErr_SetString(PyExc_ValueError, "foo"); throw py::error_already_set(); - } catch (const py::error_already_set&) { - return true; + } catch (const py::error_already_set &) { + retval = true; } - return false; + return retval; }); - m.def("python_alreadyset_in_destructor", [](py::str s) { + m.def("python_alreadyset_in_destructor", [](const py::str &s) { PythonAlreadySetInDestructor alreadyset_in_destructor(s); return true; }); // test_nested_throws - m.def("try_catch", [m](py::object exc_type, py::function f, py::args args) { - try { f(*args); } - catch (py::error_already_set &ex) { - if (ex.matches(exc_type)) - py::print(ex.what()); - else - throw; + m.def("try_catch", + [m](const py::object &exc_type, const py::function &f, const py::args &args) { + try { + f(*args); + } catch (py::error_already_set &ex) { + if (ex.matches(exc_type)) { + py::print(ex.what()); + } else { + // Simply `throw;` also works and is better, but using `throw ex;` + // here to cover that situation (as observed in the wild). + throw ex; // Invokes the copy ctor. + } + } + }); + + // Test repr that cannot be displayed + m.def("simple_bool_passthrough", [](bool x) { return x; }); + + m.def("throw_should_be_translated_to_key_error", []() { throw shared_exception(); }); + + m.def("raise_from", []() { + PyErr_SetString(PyExc_ValueError, "inner"); + py::raise_from(PyExc_ValueError, "outer"); + throw py::error_already_set(); + }); + + m.def("raise_from_already_set", []() { + try { + PyErr_SetString(PyExc_ValueError, "inner"); + throw py::error_already_set(); + } catch (py::error_already_set &e) { + py::raise_from(e, PyExc_ValueError, "outer"); + throw py::error_already_set(); } }); - // Test repr that cannot be displayed - m.def("simple_bool_passthrough", [](bool x) {return x;}); + m.def("throw_nested_exception", []() { + try { + throw std::runtime_error("Inner Exception"); + } catch (const std::runtime_error &) { + std::throw_with_nested(std::runtime_error("Outer Exception")); + } + }); + m.def("error_already_set_what", [](const py::object &exc_type, const py::object &exc_value) { + PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); + std::string what = py::error_already_set().what(); + bool py_err_set_after_what = (PyErr_Occurred() != nullptr); + PyErr_Clear(); + return py::make_tuple(std::move(what), py_err_set_after_what); + }); + + m.def("test_cross_module_interleaved_error_already_set", []() { + auto cm = py::module_::import("cross_module_interleaved_error_already_set"); + auto interleaved_error_already_set + = reinterpret_cast<void (*)()>(PyLong_AsVoidPtr(cm.attr("funcaddr").ptr())); + interleaved_error_already_set(); + }); + + m.def("test_error_already_set_double_restore", [](bool dry_run) { + PyErr_SetString(PyExc_ValueError, "Random error."); + py::error_already_set e; + e.restore(); + PyErr_Clear(); + if (!dry_run) { + e.restore(); + } + }); + + // https://github.com/pybind/pybind11/issues/4075 + m.def("test_pypy_oserror_normalization", []() { + try { + py::module_::import("io").attr("open")("this_filename_must_not_exist", "r"); + } catch (const py::error_already_set &e) { + return py::str(e.what()); // str must be built before e goes out of scope. + } + return py::str("UNEXPECTED"); + }); + + m.def("test_fn_cast_int", [](const py::function &fn) { + // function returns None instead of int, should give a useful error message + fn().cast<int>(); + }); } diff --git a/tests/test_exceptions.h b/tests/test_exceptions.h new file mode 100644 index 00000000..03684b89 --- /dev/null +++ b/tests/test_exceptions.h @@ -0,0 +1,13 @@ +#pragma once +#include "pybind11_tests.h" + +#include <stdexcept> + +// shared exceptions for cross_module_tests + +class PYBIND11_EXPORT_EXCEPTION shared_exception : public pybind11::builtin_exception { +public: + using builtin_exception::builtin_exception; + explicit shared_exception() : shared_exception("") {} + void set_error() const override { PyErr_SetString(PyExc_RuntimeError, what()); } +}; diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index c6cb6529..ccac4536 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- import sys import pytest -from pybind11_tests import exceptions as m +import env import pybind11_cross_module_tests as cm +import pybind11_tests # noqa: F401 +from pybind11_tests import exceptions as m def test_std_exception(msg): @@ -16,14 +17,31 @@ def test_std_exception(msg): def test_error_already_set(msg): with pytest.raises(RuntimeError) as excinfo: m.throw_already_set(False) - assert msg(excinfo.value) == "Unknown internal error occurred" + assert ( + msg(excinfo.value) + == "Internal error: pybind11::error_already_set called while Python error indicator not set." + ) with pytest.raises(ValueError) as excinfo: m.throw_already_set(True) assert msg(excinfo.value) == "foo" -def test_cross_module_exceptions(): +def test_raise_from(msg): + with pytest.raises(ValueError) as excinfo: + m.raise_from() + assert msg(excinfo.value) == "outer" + assert msg(excinfo.value.__cause__) == "inner" + + +def test_raise_from_already_set(msg): + with pytest.raises(ValueError) as excinfo: + m.raise_from_already_set() + assert msg(excinfo.value) == "outer" + assert msg(excinfo.value.__cause__) == "inner" + + +def test_cross_module_exceptions(msg): with pytest.raises(RuntimeError) as excinfo: cm.raise_runtime_error() assert str(excinfo.value) == "My runtime error" @@ -43,6 +61,27 @@ def test_cross_module_exceptions(): with pytest.raises(StopIteration) as excinfo: cm.throw_stop_iteration() + with pytest.raises(cm.LocalSimpleException) as excinfo: + cm.throw_local_simple_error() + assert msg(excinfo.value) == "external mod" + + with pytest.raises(KeyError) as excinfo: + cm.throw_local_error() + # KeyError is a repr of the key, so it has an extra set of quotes + assert str(excinfo.value) == "'just local'" + + +# TODO: FIXME +@pytest.mark.xfail( + "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang'))", + raises=RuntimeError, + reason="See Issue #2847, PR #2999, PR #4324", +) +def test_cross_module_exception_translator(): + with pytest.raises(KeyError): + # translator registered in cross_module_tests + m.throw_should_be_translated_to_key_error() + def test_python_call_in_catch(): d = {} @@ -53,16 +92,17 @@ def test_python_call_in_catch(): def ignore_pytest_unraisable_warning(f): unraisable = "PytestUnraisableExceptionWarning" if hasattr(pytest, unraisable): # Python >= 3.8 and pytest >= 6 - dec = pytest.mark.filterwarnings("ignore::pytest.{}".format(unraisable)) + dec = pytest.mark.filterwarnings(f"ignore::pytest.{unraisable}") return dec(f) - else: - return f + return f +# TODO: find out why this fails on PyPy, https://foss.heptapod.net/pypy/pypy/-/issues/3583 +@pytest.mark.xfail(env.PYPY, reason="Failure on PyPy 3.8 (7.3.7)", strict=False) @ignore_pytest_unraisable_warning def test_python_alreadyset_in_destructor(monkeypatch, capsys): hooked = False - triggered = [False] # mutable, so Python 2.7 closure can modify it + triggered = False if hasattr(sys, "unraisablehook"): # Python 3.8+ hooked = True @@ -72,7 +112,8 @@ def test_python_alreadyset_in_destructor(monkeypatch, capsys): def hook(unraisable_hook_args): exc_type, exc_value, exc_tb, err_msg, obj = unraisable_hook_args if obj == "already_set demo": - triggered[0] = True + nonlocal triggered + triggered = True default_hook(unraisable_hook_args) return @@ -81,11 +122,11 @@ def test_python_alreadyset_in_destructor(monkeypatch, capsys): assert m.python_alreadyset_in_destructor("already_set demo") is True if hooked: - assert triggered[0] is True + assert triggered is True _, captured_stderr = capsys.readouterr() - # Error message is different in Python 2 and 3, check for words that appear in both - assert "ignored" in captured_stderr and "already_set demo" in captured_stderr + assert captured_stderr.startswith("Exception ignored in: 'already_set demo'") + assert captured_stderr.rstrip().endswith("KeyError: 'bar'") def test_exception_matches(): @@ -141,11 +182,11 @@ def test_custom(msg): m.throws5_1() assert msg(excinfo.value) == "MyException5 subclass" - with pytest.raises(m.MyException5) as excinfo: + with pytest.raises(m.MyException5) as excinfo: # noqa: PT012 try: m.throws5() - except m.MyException5_1: - raise RuntimeError("Exception error: caught child from parent") + except m.MyException5_1 as err: + raise RuntimeError("Exception error: caught child from parent") from err assert msg(excinfo.value) == "this is a helper-defined translated exception" @@ -170,7 +211,7 @@ def test_nested_throws(capture): m.try_catch(m.MyException5, throw_myex) assert str(excinfo.value) == "nested error" - def pycatch(exctype, f, *args): + def pycatch(exctype, f, *args): # noqa: ARG001 try: f(*args) except m.MyException as e: @@ -199,11 +240,174 @@ def test_nested_throws(capture): assert str(excinfo.value) == "this is a helper-defined translated exception" +def test_throw_nested_exception(): + with pytest.raises(RuntimeError) as excinfo: + m.throw_nested_exception() + assert str(excinfo.value) == "Outer Exception" + assert str(excinfo.value.__cause__) == "Inner Exception" + + # This can often happen if you wrap a pybind11 class in a Python wrapper def test_invalid_repr(): - class MyRepr(object): + class MyRepr: def __repr__(self): raise AttributeError("Example error") with pytest.raises(TypeError): m.simple_bool_passthrough(MyRepr()) + + +def test_local_translator(msg): + """Tests that a local translator works and that the local translator from + the cross module is not applied""" + with pytest.raises(RuntimeError) as excinfo: + m.throws6() + assert msg(excinfo.value) == "MyException6 only handled in this module" + + with pytest.raises(RuntimeError) as excinfo: + m.throws_local_error() + assert not isinstance(excinfo.value, KeyError) + assert msg(excinfo.value) == "never caught" + + with pytest.raises(Exception) as excinfo: + m.throws_local_simple_error() + assert not isinstance(excinfo.value, cm.LocalSimpleException) + assert msg(excinfo.value) == "this mod" + + +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + +class FlakyException(Exception): + def __init__(self, failure_point): + if failure_point == "failure_point_init": + raise ValueError("triggered_failure_point_init") + self.failure_point = failure_point + + def __str__(self): + if self.failure_point == "failure_point_str": + raise ValueError("triggered_failure_point_str") + return "FlakyException.__str__" + + +@pytest.mark.parametrize( + ("exc_type", "exc_value", "expected_what"), + [ + (ValueError, "plain_str", "ValueError: plain_str"), + (ValueError, ("tuple_elem",), "ValueError: tuple_elem"), + (FlakyException, ("happy",), "FlakyException: FlakyException.__str__"), + ], +) +def test_error_already_set_what_with_happy_exceptions( + exc_type, exc_value, expected_what +): + what, py_err_set_after_what = m.error_already_set_what(exc_type, exc_value) + assert not py_err_set_after_what + assert what == expected_what + + +def _test_flaky_exception_failure_point_init_before_py_3_12(): + with pytest.raises(RuntimeError) as excinfo: + m.error_already_set_what(FlakyException, ("failure_point_init",)) + lines = str(excinfo.value).splitlines() + # PyErr_NormalizeException replaces the original FlakyException with ValueError: + assert lines[:3] == [ + "pybind11::error_already_set: MISMATCH of original and normalized active exception types:" + " ORIGINAL FlakyException REPLACED BY ValueError: triggered_failure_point_init", + "", + "At:", + ] + # Checking the first two lines of the traceback as formatted in error_string(): + assert "test_exceptions.py(" in lines[3] + assert lines[3].endswith("): __init__") + assert lines[4].endswith( + "): _test_flaky_exception_failure_point_init_before_py_3_12" + ) + + +def _test_flaky_exception_failure_point_init_py_3_12(): + # Behavior change in Python 3.12: https://github.com/python/cpython/issues/102594 + what, py_err_set_after_what = m.error_already_set_what( + FlakyException, ("failure_point_init",) + ) + assert not py_err_set_after_what + lines = what.splitlines() + assert lines[0].endswith("ValueError[WITH __notes__]: triggered_failure_point_init") + assert lines[1] == "__notes__ (len=1):" + assert "Normalization failed:" in lines[2] + assert "FlakyException" in lines[2] + + +@pytest.mark.skipif( + "env.PYPY and sys.version_info[:2] < (3, 12)", + reason="PyErr_NormalizeException Segmentation fault", +) +def test_flaky_exception_failure_point_init(): + if sys.version_info[:2] < (3, 12): + _test_flaky_exception_failure_point_init_before_py_3_12() + else: + _test_flaky_exception_failure_point_init_py_3_12() + + +def test_flaky_exception_failure_point_str(): + what, py_err_set_after_what = m.error_already_set_what( + FlakyException, ("failure_point_str",) + ) + assert not py_err_set_after_what + lines = what.splitlines() + n = 3 if env.PYPY and len(lines) == 3 else 5 + assert ( + lines[:n] + == [ + "FlakyException: <MESSAGE UNAVAILABLE DUE TO ANOTHER EXCEPTION>", + "", + "MESSAGE UNAVAILABLE DUE TO EXCEPTION: ValueError: triggered_failure_point_str", + "", + "At:", + ][:n] + ) + + +def test_cross_module_interleaved_error_already_set(): + with pytest.raises(RuntimeError) as excinfo: + m.test_cross_module_interleaved_error_already_set() + assert str(excinfo.value) in ( + "2nd error.", # Almost all platforms. + "RuntimeError: 2nd error.", # Some PyPy builds (seen under macOS). + ) + + +def test_error_already_set_double_restore(): + m.test_error_already_set_double_restore(True) # dry_run + with pytest.raises(RuntimeError) as excinfo: + m.test_error_already_set_double_restore(False) + assert str(excinfo.value) == ( + "Internal error: pybind11::detail::error_fetch_and_normalize::restore()" + " called a second time. ORIGINAL ERROR: ValueError: Random error." + ) + + +def test_pypy_oserror_normalization(): + # https://github.com/pybind/pybind11/issues/4075 + what = m.test_pypy_oserror_normalization() + assert "this_filename_must_not_exist" in what + + +def test_fn_cast_int_exception(): + with pytest.raises(RuntimeError) as excinfo: + m.test_fn_cast_int(lambda: None) + + assert str(excinfo.value).startswith( + "Unable to cast Python instance of type <class 'NoneType'> to C++ type" + ) diff --git a/tests/test_factory_constructors.cpp b/tests/test_factory_constructors.cpp index 7ff7e7b5..a387cd2e 100644 --- a/tests/test_factory_constructors.cpp +++ b/tests/test_factory_constructors.cpp @@ -8,35 +8,45 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include "constructor_stats.h" +#include "pybind11_tests.h" + #include <cmath> #include <new> +#include <utility> // Classes for testing python construction via C++ factory function: // Not publicly constructible, copyable, or movable: class TestFactory1 { friend class TestFactoryHelper; TestFactory1() : value("(empty)") { print_default_created(this); } - TestFactory1(int v) : value(std::to_string(v)) { print_created(this, value); } - TestFactory1(std::string v) : value(std::move(v)) { print_created(this, value); } + explicit TestFactory1(int v) : value(std::to_string(v)) { print_created(this, value); } + explicit TestFactory1(std::string v) : value(std::move(v)) { print_created(this, value); } + +public: + std::string value; TestFactory1(TestFactory1 &&) = delete; TestFactory1(const TestFactory1 &) = delete; TestFactory1 &operator=(TestFactory1 &&) = delete; TestFactory1 &operator=(const TestFactory1 &) = delete; -public: - std::string value; ~TestFactory1() { print_destroyed(this); } }; // Non-public construction, but moveable: class TestFactory2 { friend class TestFactoryHelper; TestFactory2() : value("(empty2)") { print_default_created(this); } - TestFactory2(int v) : value(std::to_string(v)) { print_created(this, value); } - TestFactory2(std::string v) : value(std::move(v)) { print_created(this, value); } + explicit TestFactory2(int v) : value(std::to_string(v)) { print_created(this, value); } + explicit TestFactory2(std::string v) : value(std::move(v)) { print_created(this, value); } + public: - TestFactory2(TestFactory2 &&m) { value = std::move(m.value); print_move_created(this); } - TestFactory2 &operator=(TestFactory2 &&m) { value = std::move(m.value); print_move_assigned(this); return *this; } + TestFactory2(TestFactory2 &&m) noexcept : value{std::move(m.value)} { + print_move_created(this); + } + TestFactory2 &operator=(TestFactory2 &&m) noexcept { + value = std::move(m.value); + print_move_assigned(this); + return *this; + } std::string value; ~TestFactory2() { print_destroyed(this); } }; @@ -45,11 +55,18 @@ class TestFactory3 { protected: friend class TestFactoryHelper; TestFactory3() : value("(empty3)") { print_default_created(this); } - TestFactory3(int v) : value(std::to_string(v)) { print_created(this, value); } + explicit TestFactory3(int v) : value(std::to_string(v)) { print_created(this, value); } + public: - TestFactory3(std::string v) : value(std::move(v)) { print_created(this, value); } - TestFactory3(TestFactory3 &&m) { value = std::move(m.value); print_move_created(this); } - TestFactory3 &operator=(TestFactory3 &&m) { value = std::move(m.value); print_move_assigned(this); return *this; } + explicit TestFactory3(std::string v) : value(std::move(v)) { print_created(this, value); } + TestFactory3(TestFactory3 &&m) noexcept : value{std::move(m.value)} { + print_move_created(this); + } + TestFactory3 &operator=(TestFactory3 &&m) noexcept { + value = std::move(m.value); + print_move_assigned(this); + return *this; + } std::string value; virtual ~TestFactory3() { print_destroyed(this); } }; @@ -57,13 +74,13 @@ public: class TestFactory4 : public TestFactory3 { public: TestFactory4() : TestFactory3() { print_default_created(this); } - TestFactory4(int v) : TestFactory3(v) { print_created(this, v); } + explicit TestFactory4(int v) : TestFactory3(v) { print_created(this, v); } ~TestFactory4() override { print_destroyed(this); } }; // Another class for an invalid downcast test class TestFactory5 : public TestFactory3 { public: - TestFactory5(int i) : TestFactory3(i) { print_created(this, i); } + explicit TestFactory5(int i) : TestFactory3(i) { print_created(this, i); } ~TestFactory5() override { print_destroyed(this); } }; @@ -71,23 +88,45 @@ class TestFactory6 { protected: int value; bool alias = false; + public: - TestFactory6(int i) : value{i} { print_created(this, i); } - TestFactory6(TestFactory6 &&f) { print_move_created(this); value = f.value; alias = f.alias; } - TestFactory6(const TestFactory6 &f) { print_copy_created(this); value = f.value; alias = f.alias; } + explicit TestFactory6(int i) : value{i} { print_created(this, i); } + TestFactory6(TestFactory6 &&f) noexcept { + print_move_created(this); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + value = f.value; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + alias = f.alias; + } + TestFactory6(const TestFactory6 &f) { + print_copy_created(this); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + value = f.value; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + alias = f.alias; + } virtual ~TestFactory6() { print_destroyed(this); } virtual int get() { return value; } - bool has_alias() { return alias; } + bool has_alias() const { return alias; } }; class PyTF6 : public TestFactory6 { public: // Special constructor that allows the factory to construct a PyTF6 from a TestFactory6 only // when an alias is needed: - PyTF6(TestFactory6 &&base) : TestFactory6(std::move(base)) { alias = true; print_created(this, "move", value); } - PyTF6(int i) : TestFactory6(i) { alias = true; print_created(this, i); } - PyTF6(PyTF6 &&f) : TestFactory6(std::move(f)) { print_move_created(this); } + explicit PyTF6(TestFactory6 &&base) : TestFactory6(std::move(base)) { + alias = true; + print_created(this, "move", value); + } + explicit PyTF6(int i) : TestFactory6(i) { + alias = true; + print_created(this, i); + } + PyTF6(PyTF6 &&f) noexcept : TestFactory6(std::move(f)) { print_move_created(this); } PyTF6(const PyTF6 &f) : TestFactory6(f) { print_copy_created(this); } - PyTF6(std::string s) : TestFactory6((int) s.size()) { alias = true; print_created(this, s); } + explicit PyTF6(std::string s) : TestFactory6((int) s.size()) { + alias = true; + print_created(this, s); + } ~PyTF6() override { print_destroyed(this); } int get() override { PYBIND11_OVERRIDE(int, TestFactory6, get, /*no args*/); } }; @@ -96,56 +135,80 @@ class TestFactory7 { protected: int value; bool alias = false; + public: - TestFactory7(int i) : value{i} { print_created(this, i); } - TestFactory7(TestFactory7 &&f) { print_move_created(this); value = f.value; alias = f.alias; } - TestFactory7(const TestFactory7 &f) { print_copy_created(this); value = f.value; alias = f.alias; } + explicit TestFactory7(int i) : value{i} { print_created(this, i); } + TestFactory7(TestFactory7 &&f) noexcept { + print_move_created(this); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + value = f.value; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + alias = f.alias; + } + TestFactory7(const TestFactory7 &f) { + print_copy_created(this); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + value = f.value; + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + alias = f.alias; + } virtual ~TestFactory7() { print_destroyed(this); } virtual int get() { return value; } - bool has_alias() { return alias; } + bool has_alias() const { return alias; } }; class PyTF7 : public TestFactory7 { public: - PyTF7(int i) : TestFactory7(i) { alias = true; print_created(this, i); } - PyTF7(PyTF7 &&f) : TestFactory7(std::move(f)) { print_move_created(this); } + explicit PyTF7(int i) : TestFactory7(i) { + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) + alias = true; + print_created(this, i); + } + PyTF7(PyTF7 &&f) noexcept : TestFactory7(std::move(f)) { print_move_created(this); } PyTF7(const PyTF7 &f) : TestFactory7(f) { print_copy_created(this); } ~PyTF7() override { print_destroyed(this); } int get() override { PYBIND11_OVERRIDE(int, TestFactory7, get, /*no args*/); } }; - class TestFactoryHelper { public: // Non-movable, non-copyable type: // Return via pointer: static TestFactory1 *construct1() { return new TestFactory1(); } // Holder: - static std::unique_ptr<TestFactory1> construct1(int a) { return std::unique_ptr<TestFactory1>(new TestFactory1(a)); } + static std::unique_ptr<TestFactory1> construct1(int a) { + return std::unique_ptr<TestFactory1>(new TestFactory1(a)); + } // pointer again - static TestFactory1 *construct1_string(std::string a) { return new TestFactory1(a); } + static TestFactory1 *construct1_string(std::string a) { + return new TestFactory1(std::move(a)); + } // Moveable type: // pointer: static TestFactory2 *construct2() { return new TestFactory2(); } // holder: - static std::unique_ptr<TestFactory2> construct2(int a) { return std::unique_ptr<TestFactory2>(new TestFactory2(a)); } + static std::unique_ptr<TestFactory2> construct2(int a) { + return std::unique_ptr<TestFactory2>(new TestFactory2(a)); + } // by value moving: - static TestFactory2 construct2(std::string a) { return TestFactory2(a); } + static TestFactory2 construct2(std::string a) { return TestFactory2(std::move(a)); } // shared_ptr holder type: // pointer: static TestFactory3 *construct3() { return new TestFactory3(); } // holder: - static std::shared_ptr<TestFactory3> construct3(int a) { return std::shared_ptr<TestFactory3>(new TestFactory3(a)); } + static std::shared_ptr<TestFactory3> construct3(int a) { + return std::shared_ptr<TestFactory3>(new TestFactory3(a)); + } }; TEST_SUBMODULE(factory_constructors, m) { // Define various trivial types to allow simpler overload resolution: py::module_ m_tag = m.def_submodule("tag"); -#define MAKE_TAG_TYPE(Name) \ - struct Name##_tag {}; \ - py::class_<Name##_tag>(m_tag, #Name "_tag").def(py::init<>()); \ +#define MAKE_TAG_TYPE(Name) \ + struct Name##_tag {}; \ + py::class_<Name##_tag>(m_tag, #Name "_tag").def(py::init<>()); \ m_tag.attr(#Name) = py::cast(Name##_tag{}) MAKE_TAG_TYPE(pointer); MAKE_TAG_TYPE(unique_ptr); @@ -168,19 +231,23 @@ TEST_SUBMODULE(factory_constructors, m) { .def(py::init([](unique_ptr_tag, int v) { return TestFactoryHelper::construct1(v); })) .def(py::init(&TestFactoryHelper::construct1_string)) // raw function pointer .def(py::init([](pointer_tag) { return TestFactoryHelper::construct1(); })) - .def(py::init([](py::handle, int v, py::handle) { return TestFactoryHelper::construct1(v); })) - .def_readwrite("value", &TestFactory1::value) - ; + .def(py::init( + [](py::handle, int v, py::handle) { return TestFactoryHelper::construct1(v); })) + .def_readwrite("value", &TestFactory1::value); py::class_<TestFactory2>(m, "TestFactory2") .def(py::init([](pointer_tag, int v) { return TestFactoryHelper::construct2(v); })) - .def(py::init([](unique_ptr_tag, std::string v) { return TestFactoryHelper::construct2(v); })) + .def(py::init([](unique_ptr_tag, std::string v) { + return TestFactoryHelper::construct2(std::move(v)); + })) .def(py::init([](move_tag) { return TestFactoryHelper::construct2(); })) - .def_readwrite("value", &TestFactory2::value) - ; + .def_readwrite("value", &TestFactory2::value); // Stateful & reused: int c = 1; - auto c4a = [c](pointer_tag, TF4_tag, int a) { (void) c; return new TestFactory4(a);}; + auto c4a = [c](pointer_tag, TF4_tag, int a) { + (void) c; + return new TestFactory4(a); + }; // test_init_factory_basic, test_init_factory_casting py::class_<TestFactory3, std::shared_ptr<TestFactory3>> pyTestFactory3(m, "TestFactory3"); @@ -188,23 +255,26 @@ TEST_SUBMODULE(factory_constructors, m) { .def(py::init([](pointer_tag, int v) { return TestFactoryHelper::construct3(v); })) .def(py::init([](shared_ptr_tag) { return TestFactoryHelper::construct3(); })); ignoreOldStyleInitWarnings([&pyTestFactory3]() { - pyTestFactory3.def("__init__", [](TestFactory3 &self, std::string v) { new (&self) TestFactory3(v); }); // placement-new ctor + pyTestFactory3.def("__init__", [](TestFactory3 &self, std::string v) { + new (&self) TestFactory3(std::move(v)); + }); // placement-new ctor }); pyTestFactory3 // factories returning a derived type: .def(py::init(c4a)) // derived ptr .def(py::init([](pointer_tag, TF5_tag, int a) { return new TestFactory5(a); })) // derived shared ptr: - .def(py::init([](shared_ptr_tag, TF4_tag, int a) { return std::make_shared<TestFactory4>(a); })) - .def(py::init([](shared_ptr_tag, TF5_tag, int a) { return std::make_shared<TestFactory5>(a); })) + .def(py::init( + [](shared_ptr_tag, TF4_tag, int a) { return std::make_shared<TestFactory4>(a); })) + .def(py::init( + [](shared_ptr_tag, TF5_tag, int a) { return std::make_shared<TestFactory5>(a); })) // Returns nullptr: .def(py::init([](null_ptr_tag) { return (TestFactory3 *) nullptr; })) .def(py::init([](null_unique_ptr_tag) { return std::unique_ptr<TestFactory3>(); })) .def(py::init([](null_shared_ptr_tag) { return std::shared_ptr<TestFactory3>(); })) - .def_readwrite("value", &TestFactory3::value) - ; + .def_readwrite("value", &TestFactory3::value); // test_init_factory_casting py::class_<TestFactory4, TestFactory3, std::shared_ptr<TestFactory4>>(m, "TestFactory4") @@ -219,58 +289,60 @@ TEST_SUBMODULE(factory_constructors, m) { py::class_<TestFactory6, PyTF6>(m, "TestFactory6") .def(py::init([](base_tag, int i) { return TestFactory6(i); })) .def(py::init([](alias_tag, int i) { return PyTF6(i); })) - .def(py::init([](alias_tag, std::string s) { return PyTF6(s); })) + .def(py::init([](alias_tag, std::string s) { return PyTF6(std::move(s)); })) .def(py::init([](alias_tag, pointer_tag, int i) { return new PyTF6(i); })) .def(py::init([](base_tag, pointer_tag, int i) { return new TestFactory6(i); })) - .def(py::init([](base_tag, alias_tag, pointer_tag, int i) { return (TestFactory6 *) new PyTF6(i); })) + .def(py::init( + [](base_tag, alias_tag, pointer_tag, int i) { return (TestFactory6 *) new PyTF6(i); })) .def("get", &TestFactory6::get) .def("has_alias", &TestFactory6::has_alias) - .def_static("get_cstats", &ConstructorStats::get<TestFactory6>, py::return_value_policy::reference) - .def_static("get_alias_cstats", &ConstructorStats::get<PyTF6>, py::return_value_policy::reference) - ; + .def_static( + "get_cstats", &ConstructorStats::get<TestFactory6>, py::return_value_policy::reference) + .def_static( + "get_alias_cstats", &ConstructorStats::get<PyTF6>, py::return_value_policy::reference); // test_init_factory_dual // Separate alias constructor testing py::class_<TestFactory7, PyTF7, std::shared_ptr<TestFactory7>>(m, "TestFactory7") - .def(py::init( - [](int i) { return TestFactory7(i); }, - [](int i) { return PyTF7(i); })) - .def(py::init( - [](pointer_tag, int i) { return new TestFactory7(i); }, - [](pointer_tag, int i) { return new PyTF7(i); })) - .def(py::init( - [](mixed_tag, int i) { return new TestFactory7(i); }, - [](mixed_tag, int i) { return PyTF7(i); })) - .def(py::init( - [](mixed_tag, std::string s) { return TestFactory7((int) s.size()); }, - [](mixed_tag, std::string s) { return new PyTF7((int) s.size()); })) - .def(py::init( - [](base_tag, pointer_tag, int i) { return new TestFactory7(i); }, - [](base_tag, pointer_tag, int i) { return (TestFactory7 *) new PyTF7(i); })) - .def(py::init( - [](alias_tag, pointer_tag, int i) { return new PyTF7(i); }, - [](alias_tag, pointer_tag, int i) { return new PyTF7(10*i); })) + .def(py::init([](int i) { return TestFactory7(i); }, [](int i) { return PyTF7(i); })) + .def(py::init([](pointer_tag, int i) { return new TestFactory7(i); }, + [](pointer_tag, int i) { return new PyTF7(i); })) + .def(py::init([](mixed_tag, int i) { return new TestFactory7(i); }, + [](mixed_tag, int i) { return PyTF7(i); })) + .def(py::init([](mixed_tag, const std::string &s) { return TestFactory7((int) s.size()); }, + [](mixed_tag, const std::string &s) { return new PyTF7((int) s.size()); })) + .def(py::init([](base_tag, pointer_tag, int i) { return new TestFactory7(i); }, + [](base_tag, pointer_tag, int i) { return (TestFactory7 *) new PyTF7(i); })) + .def(py::init([](alias_tag, pointer_tag, int i) { return new PyTF7(i); }, + [](alias_tag, pointer_tag, int i) { return new PyTF7(10 * i); })) .def(py::init( [](shared_ptr_tag, base_tag, int i) { return std::make_shared<TestFactory7>(i); }, - [](shared_ptr_tag, base_tag, int i) { auto *p = new PyTF7(i); return std::shared_ptr<TestFactory7>(p); })) - .def(py::init( - [](shared_ptr_tag, invalid_base_tag, int i) { return std::make_shared<TestFactory7>(i); }, - [](shared_ptr_tag, invalid_base_tag, int i) { return std::make_shared<TestFactory7>(i); })) // <-- invalid alias factory + [](shared_ptr_tag, base_tag, int i) { + auto *p = new PyTF7(i); + return std::shared_ptr<TestFactory7>(p); + })) + .def(py::init([](shared_ptr_tag, + invalid_base_tag, + int i) { return std::make_shared<TestFactory7>(i); }, + [](shared_ptr_tag, invalid_base_tag, int i) { + return std::make_shared<TestFactory7>(i); + })) // <-- invalid alias factory .def("get", &TestFactory7::get) .def("has_alias", &TestFactory7::has_alias) - .def_static("get_cstats", &ConstructorStats::get<TestFactory7>, py::return_value_policy::reference) - .def_static("get_alias_cstats", &ConstructorStats::get<PyTF7>, py::return_value_policy::reference) - ; + .def_static( + "get_cstats", &ConstructorStats::get<TestFactory7>, py::return_value_policy::reference) + .def_static( + "get_alias_cstats", &ConstructorStats::get<PyTF7>, py::return_value_policy::reference); // test_placement_new_alternative // Class with a custom new operator but *without* a placement new operator (issue #948) class NoPlacementNew { public: - NoPlacementNew(int i) : i(i) { } + explicit NoPlacementNew(int i) : i(i) {} static void *operator new(std::size_t s) { auto *p = ::operator new(s); py::print("operator new called, returning", reinterpret_cast<uintptr_t>(p)); @@ -286,35 +358,39 @@ TEST_SUBMODULE(factory_constructors, m) { py::class_<NoPlacementNew>(m, "NoPlacementNew") .def(py::init<int>()) .def(py::init([]() { return new NoPlacementNew(100); })) - .def_readwrite("i", &NoPlacementNew::i) - ; - + .def_readwrite("i", &NoPlacementNew::i); // test_reallocations // Class that has verbose operator_new/operator_delete calls struct NoisyAlloc { NoisyAlloc(const NoisyAlloc &) = default; - NoisyAlloc(int i) { py::print(py::str("NoisyAlloc(int {})").format(i)); } - NoisyAlloc(double d) { py::print(py::str("NoisyAlloc(double {})").format(d)); } + explicit NoisyAlloc(int i) { py::print(py::str("NoisyAlloc(int {})").format(i)); } + explicit NoisyAlloc(double d) { py::print(py::str("NoisyAlloc(double {})").format(d)); } ~NoisyAlloc() { py::print("~NoisyAlloc()"); } - static void *operator new(size_t s) { py::print("noisy new"); return ::operator new(s); } - static void *operator new(size_t, void *p) { py::print("noisy placement new"); return p; } - static void operator delete(void *p, size_t) { py::print("noisy delete"); ::operator delete(p); } + static void *operator new(size_t s) { + py::print("noisy new"); + return ::operator new(s); + } + static void *operator new(size_t, void *p) { + py::print("noisy placement new"); + return p; + } + static void operator delete(void *p, size_t) { + py::print("noisy delete"); + ::operator delete(p); + } static void operator delete(void *, void *) { py::print("noisy placement delete"); } -#if defined(_MSC_VER) && _MSC_VER < 1910 - // MSVC 2015 bug: the above "noisy delete" isn't invoked (fixed in MSVC 2017) - static void operator delete(void *p) { py::print("noisy delete"); ::operator delete(p); } -#endif }; - py::class_<NoisyAlloc> pyNoisyAlloc(m, "NoisyAlloc"); - // Since these overloads have the same number of arguments, the dispatcher will try each of - // them until the arguments convert. Thus we can get a pre-allocation here when passing a - // single non-integer: + // Since these overloads have the same number of arguments, the dispatcher will try each of + // them until the arguments convert. Thus we can get a pre-allocation here when passing a + // single non-integer: ignoreOldStyleInitWarnings([&pyNoisyAlloc]() { - pyNoisyAlloc.def("__init__", [](NoisyAlloc *a, int i) { new (a) NoisyAlloc(i); }); // Regular constructor, runs first, requires preallocation + pyNoisyAlloc.def("__init__", [](NoisyAlloc *a, int i) { + new (a) NoisyAlloc(i); + }); // Regular constructor, runs first, requires preallocation }); pyNoisyAlloc.def(py::init([](double d) { return new NoisyAlloc(d); })); @@ -325,19 +401,19 @@ TEST_SUBMODULE(factory_constructors, m) { pyNoisyAlloc.def(py::init([](double d, int) { return NoisyAlloc(d); })); // Old-style placement new init; requires preallocation ignoreOldStyleInitWarnings([&pyNoisyAlloc]() { - pyNoisyAlloc.def("__init__", [](NoisyAlloc &a, double d, double) { new (&a) NoisyAlloc(d); }); + pyNoisyAlloc.def("__init__", + [](NoisyAlloc &a, double d, double) { new (&a) NoisyAlloc(d); }); }); // Requires deallocation of previous overload preallocated value: pyNoisyAlloc.def(py::init([](int i, double) { return new NoisyAlloc(i); })); // Regular again: requires yet another preallocation ignoreOldStyleInitWarnings([&pyNoisyAlloc]() { - pyNoisyAlloc.def("__init__", [](NoisyAlloc &a, int i, std::string) { new (&a) NoisyAlloc(i); }); + pyNoisyAlloc.def( + "__init__", [](NoisyAlloc &a, int i, const std::string &) { new (&a) NoisyAlloc(i); }); }); - - - - // static_assert testing (the following def's should all fail with appropriate compilation errors): + // static_assert testing (the following def's should all fail with appropriate compilation + // errors): #if 0 struct BadF1Base {}; struct BadF1 : BadF1Base {}; diff --git a/tests/test_factory_constructors.py b/tests/test_factory_constructors.py index ffcce6fd..04df8026 100644 --- a/tests/test_factory_constructors.py +++ b/tests/test_factory_constructors.py @@ -1,12 +1,10 @@ -# -*- coding: utf-8 -*- -import pytest import re -import env # noqa: F401 +import pytest +from pybind11_tests import ConstructorStats from pybind11_tests import factory_constructors as m from pybind11_tests.factory_constructors import tag -from pybind11_tests import ConstructorStats def test_init_factory_basic(): @@ -82,7 +80,7 @@ def test_init_factory_signature(msg): 4. m.factory_constructors.TestFactory1(arg0: handle, arg1: int, arg2: handle) Invoked with: 'invalid', 'constructor', 'arguments' - """ # noqa: E501 line too long + """ ) assert ( @@ -98,7 +96,7 @@ def test_init_factory_signature(msg): 3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None 4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None - """ # noqa: E501 line too long + """ ) @@ -465,12 +463,10 @@ def test_reallocation_g(capture, msg): ) -@pytest.mark.skipif("env.PY2") def test_invalid_self(): - """Tests invocation of the pybind-registered base class with an invalid `self` argument. You - can only actually do this on Python 3: Python 2 raises an exception itself if you try.""" + """Tests invocation of the pybind-registered base class with an invalid `self` argument.""" - class NotPybindDerived(object): + class NotPybindDerived: pass # Attempts to initialize with an invalid type passed as `self`: @@ -486,7 +482,9 @@ def test_invalid_self(): # Same as above, but for a class with an alias: class BrokenTF6(m.TestFactory6): def __init__(self, bad): - if bad == 1: + if bad == 0: + m.TestFactory6.__init__() + elif bad == 1: a = m.TestFactory2(tag.pointer, 1) m.TestFactory6.__init__(a, tag.base, 1) elif bad == 2: @@ -506,13 +504,13 @@ def test_invalid_self(): BrokenTF1(arg) assert ( str(excinfo.value) - == "__init__(self, ...) called with invalid `self` argument" + == "__init__(self, ...) called with invalid or missing `self` argument" ) - for arg in (1, 2, 3, 4): + for arg in (0, 1, 2, 3, 4): with pytest.raises(TypeError) as excinfo: BrokenTF6(arg) assert ( str(excinfo.value) - == "__init__(self, ...) called with invalid `self` argument" + == "__init__(self, ...) called with invalid or missing `self` argument" ) diff --git a/tests/test_gil_scoped.cpp b/tests/test_gil_scoped.cpp index b6a45a5f..f136086e 100644 --- a/tests/test_gil_scoped.cpp +++ b/tests/test_gil_scoped.cpp @@ -7,48 +7,138 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/functional.h> +#include "pybind11_tests.h" + +#include <string> +#include <thread> + +#define CROSS_MODULE(Function) \ + auto cm = py::module_::import("cross_module_gil_utils"); \ + auto target = reinterpret_cast<void (*)()>(PyLong_AsVoidPtr(cm.attr(Function).ptr())); -class VirtClass { +class VirtClass { public: virtual ~VirtClass() = default; VirtClass() = default; - VirtClass(const VirtClass&) = delete; + VirtClass(const VirtClass &) = delete; virtual void virtual_func() {} virtual void pure_virtual_func() = 0; }; class PyVirtClass : public VirtClass { - void virtual_func() override { - PYBIND11_OVERRIDE(void, VirtClass, virtual_func,); - } + void virtual_func() override { PYBIND11_OVERRIDE(void, VirtClass, virtual_func, ); } void pure_virtual_func() override { - PYBIND11_OVERRIDE_PURE(void, VirtClass, pure_virtual_func,); + PYBIND11_OVERRIDE_PURE(void, VirtClass, pure_virtual_func, ); } }; TEST_SUBMODULE(gil_scoped, m) { - py::class_<VirtClass, PyVirtClass>(m, "VirtClass") - .def(py::init<>()) - .def("virtual_func", &VirtClass::virtual_func) - .def("pure_virtual_func", &VirtClass::pure_virtual_func); - - m.def("test_callback_py_obj", - [](py::object func) { func(); }); - m.def("test_callback_std_func", - [](const std::function<void()> &func) { func(); }); - m.def("test_callback_virtual_func", - [](VirtClass &virt) { virt.virtual_func(); }); - m.def("test_callback_pure_virtual_func", - [](VirtClass &virt) { virt.pure_virtual_func(); }); - m.def("test_cross_module_gil", - []() { - auto cm = py::module_::import("cross_module_gil_utils"); - auto gil_acquire = reinterpret_cast<void (*)()>( - PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); - py::gil_scoped_release gil_release; - gil_acquire(); - }); + m.attr("defined_THREAD_SANITIZER") = +#if defined(THREAD_SANITIZER) + true; +#else + false; +#endif + + m.def("intentional_deadlock", + []() { std::thread([]() { py::gil_scoped_acquire gil_acquired; }).join(); }); + + py::class_<VirtClass, PyVirtClass>(m, "VirtClass") + .def(py::init<>()) + .def("virtual_func", &VirtClass::virtual_func) + .def("pure_virtual_func", &VirtClass::pure_virtual_func); + + m.def("test_callback_py_obj", [](py::object &func) { func(); }); + m.def("test_callback_std_func", [](const std::function<void()> &func) { func(); }); + m.def("test_callback_virtual_func", [](VirtClass &virt) { virt.virtual_func(); }); + m.def("test_callback_pure_virtual_func", [](VirtClass &virt) { virt.pure_virtual_func(); }); + m.def("test_cross_module_gil_released", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_acquired", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_custom_released", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_custom_acquired", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_released", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_custom_released", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_custom_acquired", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_released", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_release_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired; + return py::str(obj); + }); + m.def("test_nested_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired_outer; + py::gil_scoped_acquire gil_acquired_inner; + return py::str(obj); + }); + m.def("test_multi_acquire_release_cross_module", [](unsigned bits) { + py::set internals_ids; + internals_ids.add(PYBIND11_INTERNALS_ID); + { + py::gil_scoped_release gil_released; + auto thread_f = [bits, &internals_ids]() { + py::gil_scoped_acquire gil_acquired; + auto cm = py::module_::import("cross_module_gil_utils"); + auto target = reinterpret_cast<std::string (*)(unsigned)>( + PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr())); + std::string cm_internals_id = target(bits >> 3); + internals_ids.add(cm_internals_id); + }; + if ((bits & 0x1u) != 0u) { + thread_f(); + } + if ((bits & 0x2u) != 0u) { + std::thread non_python_thread(thread_f); + non_python_thread.join(); + } + if ((bits & 0x4u) != 0u) { + thread_f(); + } + } + return internals_ids; + }); } diff --git a/tests/test_gil_scoped.py b/tests/test_gil_scoped.py index 0a1d6274..fc8af9b7 100644 --- a/tests/test_gil_scoped.py +++ b/tests/test_gil_scoped.py @@ -1,46 +1,197 @@ -# -*- coding: utf-8 -*- import multiprocessing +import sys import threading +import time +import pytest + +import env from pybind11_tests import gil_scoped as m +class ExtendedVirtClass(m.VirtClass): + def virtual_func(self): + pass + + def pure_virtual_func(self): + pass + + +def test_callback_py_obj(): + m.test_callback_py_obj(lambda: None) + + +def test_callback_std_func(): + m.test_callback_std_func(lambda: None) + + +def test_callback_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_virtual_func(extended) + + +def test_callback_pure_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_pure_virtual_func(extended) + + +def test_cross_module_gil_released(): + """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" + m.test_cross_module_gil_released() # Should not raise a SIGSEGV + + +def test_cross_module_gil_acquired(): + """Makes sure that the GIL can be acquired by another module from a GIL-acquired state.""" + m.test_cross_module_gil_acquired() # Should not raise a SIGSEGV + + +def test_cross_module_gil_inner_custom_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_released() + + +def test_cross_module_gil_inner_custom_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_acquired() + + +def test_cross_module_gil_inner_pybind11_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_released() + + +def test_cross_module_gil_inner_pybind11_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_acquired() + + +def test_cross_module_gil_nested_custom_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_released() + + +def test_cross_module_gil_nested_custom_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_acquired() + + +def test_cross_module_gil_nested_pybind11_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_released() + + +def test_cross_module_gil_nested_pybind11_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_acquired() + + +def test_release_acquire(): + assert m.test_release_acquire(0xAB) == "171" + + +def test_nested_acquire(): + assert m.test_nested_acquire(0xAB) == "171" + + +def test_multi_acquire_release_cross_module(): + for bits in range(16 * 8): + internals_ids = m.test_multi_acquire_release_cross_module(bits) + assert len(internals_ids) == 2 if bits % 8 else 1 + + +# Intentionally putting human review in the loop here, to guard against accidents. +VARS_BEFORE_ALL_BASIC_TESTS = dict(vars()) # Make a copy of the dict (critical). +ALL_BASIC_TESTS = ( + test_callback_py_obj, + test_callback_std_func, + test_callback_virtual_func, + test_callback_pure_virtual_func, + test_cross_module_gil_released, + test_cross_module_gil_acquired, + test_cross_module_gil_inner_custom_released, + test_cross_module_gil_inner_custom_acquired, + test_cross_module_gil_inner_pybind11_released, + test_cross_module_gil_inner_pybind11_acquired, + test_cross_module_gil_nested_custom_released, + test_cross_module_gil_nested_custom_acquired, + test_cross_module_gil_nested_pybind11_released, + test_cross_module_gil_nested_pybind11_acquired, + test_release_acquire, + test_nested_acquire, + test_multi_acquire_release_cross_module, +) + + +def test_all_basic_tests_completeness(): + num_found = 0 + for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items(): + if not key.startswith("test_"): + continue + assert value in ALL_BASIC_TESTS + num_found += 1 + assert len(ALL_BASIC_TESTS) == num_found + + +def _intentional_deadlock(): + m.intentional_deadlock() + + +ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) + + def _run_in_process(target, *args, **kwargs): - """Runs target in process and returns its exitcode after 10s (None if still alive).""" + test_fn = target if len(args) == 0 else args[0] + # Do not need to wait much, 10s should be more than enough. + timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: + t_start = time.time() process.start() - # Do not need to wait much, 10s should be more than enough. - process.join(timeout=10) + if timeout >= 100: # For debugging. + print( + "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs) + ) + print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True) + process.join(timeout=timeout) + if timeout >= 100: + print("\nprocess.pid JOINED", process.pid, flush=True) + t_delta = time.time() - t_start + if process.exitcode == 66 and m.defined_THREAD_SANITIZER: # Issue #2754 + # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output. + # Maybe this could work: + # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e + pytest.skip( + "ThreadSanitizer: starting new threads after multi-threaded fork is not supported." + ) + elif test_fn is _intentional_deadlock: + assert process.exitcode is None + return 0 + + if process.exitcode is None: + assert t_delta > 0.9 * timeout + msg = "DEADLOCK, most likely, exactly what this test is meant to detect." + if env.PYPY and env.WIN: + pytest.skip(msg) + raise RuntimeError(msg) return process.exitcode finally: if process.is_alive(): process.terminate() -def _python_to_cpp_to_python(): - """Calls different C++ functions that come back to Python.""" - - class ExtendedVirtClass(m.VirtClass): - def virtual_func(self): - pass - - def pure_virtual_func(self): - pass - - extended = ExtendedVirtClass() - m.test_callback_py_obj(lambda: None) - m.test_callback_std_func(lambda: None) - m.test_callback_virtual_func(extended) - m.test_callback_pure_virtual_func(extended) - - -def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): - """Calls different C++ functions that come back to Python, from Python threads.""" +def _run_in_threads(test_fn, num_threads, parallel): threads = [] for _ in range(num_threads): - thread = threading.Thread(target=_python_to_cpp_to_python) + thread = threading.Thread(target=test_fn) thread.daemon = True thread.start() if parallel: @@ -52,43 +203,40 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_parallel(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_sequential(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert ( - _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 - ) + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_process(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. This test is for completion, but it was never an issue. """ - assert _run_in_process(_python_to_cpp_to_python) == 0 - - -def test_cross_module_gil(): - """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" - m.test_cross_module_gil() # Should not raise a SIGSEGV + assert _run_in_process(test_fn) == 0 diff --git a/tests/test_iostream.cpp b/tests/test_iostream.cpp index 1be0655d..421eaa2d 100644 --- a/tests/test_iostream.cpp +++ b/tests/test_iostream.cpp @@ -7,25 +7,25 @@ BSD-style license that can be found in the LICENSE file. */ -#if defined(_MSC_VER) && _MSC_VER < 1910 // VS 2015's MSVC -# pragma warning(disable: 4702) // unreachable code in system header (xatomic.h(382)) -#endif - #include <pybind11/iostream.h> + #include "pybind11_tests.h" + #include <atomic> #include <iostream> +#include <mutex> +#include <string> #include <thread> - -void noisy_function(std::string msg, bool flush) { +void noisy_function(const std::string &msg, bool flush) { std::cout << msg; - if (flush) + if (flush) { std::cout << std::flush; + } } -void noisy_funct_dual(std::string msg, std::string emsg) { +void noisy_funct_dual(const std::string &msg, const std::string &emsg) { std::cout << msg; std::cerr << emsg; } @@ -34,22 +34,31 @@ void noisy_funct_dual(std::string msg, std::string emsg) { // simply repeatedly write to std::cerr until stopped // redirect is called at some point to test the safety of scoped_estream_redirect struct TestThread { - TestThread() : t_{nullptr}, stop_{false} { + TestThread() : stop_{false} { auto thread_f = [this] { + static std::mutex cout_mutex; while (!stop_) { - std::cout << "x" << std::flush; + { + // #HelpAppreciated: Work on iostream.h thread safety. + // Without this lock, the clang ThreadSanitizer (tsan) reliably reports a + // data race, and this test is predictably flakey on Windows. + // For more background see the discussion under + // https://github.com/pybind/pybind11/pull/2982 and + // https://github.com/pybind/pybind11/pull/2995. + const std::lock_guard<std::mutex> lock(cout_mutex); + std::cout << "x" << std::flush; + } std::this_thread::sleep_for(std::chrono::microseconds(50)); - } }; + } + }; t_ = new std::thread(std::move(thread_f)); } - ~TestThread() { - delete t_; - } + ~TestThread() { delete t_; } void stop() { stop_ = true; } - void join() { + void join() const { py::gil_scoped_release gil_lock; t_->join(); } @@ -59,51 +68,50 @@ struct TestThread { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } - std::thread * t_; + std::thread *t_{nullptr}; std::atomic<bool> stop_; }; - TEST_SUBMODULE(iostream, m) { add_ostream_redirect(m); // test_evals - m.def("captured_output_default", [](std::string msg) { + m.def("captured_output_default", [](const std::string &msg) { py::scoped_ostream_redirect redir; std::cout << msg << std::flush; }); - m.def("captured_output", [](std::string msg) { + m.def("captured_output", [](const std::string &msg) { py::scoped_ostream_redirect redir(std::cout, py::module_::import("sys").attr("stdout")); std::cout << msg << std::flush; }); - m.def("guard_output", &noisy_function, - py::call_guard<py::scoped_ostream_redirect>(), - py::arg("msg"), py::arg("flush")=true); + m.def("guard_output", + &noisy_function, + py::call_guard<py::scoped_ostream_redirect>(), + py::arg("msg"), + py::arg("flush") = true); - m.def("captured_err", [](std::string msg) { + m.def("captured_err", [](const std::string &msg) { py::scoped_ostream_redirect redir(std::cerr, py::module_::import("sys").attr("stderr")); std::cerr << msg << std::flush; }); m.def("noisy_function", &noisy_function, py::arg("msg"), py::arg("flush") = true); - m.def("dual_guard", &noisy_funct_dual, - py::call_guard<py::scoped_ostream_redirect, py::scoped_estream_redirect>(), - py::arg("msg"), py::arg("emsg")); + m.def("dual_guard", + &noisy_funct_dual, + py::call_guard<py::scoped_ostream_redirect, py::scoped_estream_redirect>(), + py::arg("msg"), + py::arg("emsg")); - m.def("raw_output", [](std::string msg) { - std::cout << msg << std::flush; - }); + m.def("raw_output", [](const std::string &msg) { std::cout << msg << std::flush; }); - m.def("raw_err", [](std::string msg) { - std::cerr << msg << std::flush; - }); + m.def("raw_err", [](const std::string &msg) { std::cerr << msg << std::flush; }); - m.def("captured_dual", [](std::string msg, std::string emsg) { + m.def("captured_dual", [](const std::string &msg, const std::string &emsg) { py::scoped_ostream_redirect redirout(std::cout, py::module_::import("sys").attr("stdout")); py::scoped_ostream_redirect redirerr(std::cerr, py::module_::import("sys").attr("stderr")); std::cout << msg << std::flush; diff --git a/tests/test_iostream.py b/tests/test_iostream.py index 6d493bed..d283eb15 100644 --- a/tests/test_iostream.py +++ b/tests/test_iostream.py @@ -1,43 +1,7 @@ -# -*- coding: utf-8 -*- +from contextlib import redirect_stderr, redirect_stdout +from io import StringIO + from pybind11_tests import iostream as m -import sys - -from contextlib import contextmanager - -try: - # Python 3 - from io import StringIO -except ImportError: - # Python 2 - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - -try: - # Python 3.4 - from contextlib import redirect_stdout -except ImportError: - - @contextmanager - def redirect_stdout(target): - original = sys.stdout - sys.stdout = target - yield - sys.stdout = original - - -try: - # Python 3.5 - from contextlib import redirect_stderr -except ImportError: - - @contextmanager - def redirect_stderr(target): - original = sys.stderr - sys.stderr = target - yield - sys.stderr = original def test_captured(capsys): @@ -45,16 +9,16 @@ def test_captured(capsys): m.captured_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_err(msg) stdout, stderr = capsys.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg @@ -66,7 +30,97 @@ def test_captured_large_string(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr + + +def test_captured_utf8_2byte_offset0(capsys): + msg = "\u07FF" + msg = "" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_2byte_offset1(capsys): + msg = "\u07FF" + msg = "1" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_3byte_offset0(capsys): + msg = "\uFFFF" + msg = "" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_3byte_offset1(capsys): + msg = "\uFFFF" + msg = "1" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_3byte_offset2(capsys): + msg = "\uFFFF" + msg = "12" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_4byte_offset0(capsys): + msg = "\U0010FFFF" + msg = "" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_4byte_offset1(capsys): + msg = "\U0010FFFF" + msg = "1" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_4byte_offset2(capsys): + msg = "\U0010FFFF" + msg = "12" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr + + +def test_captured_utf8_4byte_offset3(capsys): + msg = "\U0010FFFF" + msg = "123" + msg * (1024 // len(msg) + 1) + + m.captured_output_default(msg) + stdout, stderr = capsys.readouterr() + assert stdout == msg + assert not stderr def test_guard_capture(capsys): @@ -74,7 +128,7 @@ def test_guard_capture(capsys): m.guard_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_series_captured(capture): @@ -91,7 +145,7 @@ def test_flush(capfd): with m.ostream_redirect(): m.noisy_function(msg, flush=False) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout m.noisy_function(msg2, flush=True) stdout, stderr = capfd.readouterr() @@ -110,15 +164,15 @@ def test_not_captured(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" - assert stream.getvalue() == "" + assert not stderr + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream): m.captured_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -128,16 +182,16 @@ def test_err(capfd): with redirect_stderr(stream): m.raw_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stderr(stream): m.captured_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -167,14 +221,13 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() - with redirect_stdout(stream): - with m.ostream_redirect(): - m.raw_output(msg) + with redirect_stdout(stream), m.ostream_redirect(): + m.raw_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stream.getvalue() == msg stream = StringIO() @@ -182,7 +235,7 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() def test_redirect_err(capfd): @@ -190,13 +243,12 @@ def test_redirect_err(capfd): msg2 = "StdErr" stream = StringIO() - with redirect_stderr(stream): - with m.ostream_redirect(stdout=False): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stderr(stream), m.ostream_redirect(stdout=False): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr assert stream.getvalue() == msg2 @@ -206,14 +258,12 @@ def test_redirect_both(capfd): stream = StringIO() stream2 = StringIO() - with redirect_stdout(stream): - with redirect_stderr(stream2): - with m.ostream_redirect(): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stdout(stream), redirect_stderr(stream2), m.ostream_redirect(): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg assert stream2.getvalue() == msg2 diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 627a7969..77e72c0c 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -7,136 +7,275 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/stl.h> +#include "constructor_stats.h" +#include "pybind11_tests.h" + +#include <utility> + TEST_SUBMODULE(kwargs_and_defaults, m) { - auto kw_func = [](int x, int y) { return "x=" + std::to_string(x) + ", y=" + std::to_string(y); }; + auto kw_func + = [](int x, int y) { return "x=" + std::to_string(x) + ", y=" + std::to_string(y); }; // test_named_arguments m.def("kw_func0", kw_func); m.def("kw_func1", kw_func, py::arg("x"), py::arg("y")); m.def("kw_func2", kw_func, py::arg("x") = 100, py::arg("y") = 200); - m.def("kw_func3", [](const char *) { }, py::arg("data") = std::string("Hello world!")); + m.def( + "kw_func3", [](const char *) {}, py::arg("data") = std::string("Hello world!")); /* A fancier default argument */ std::vector<int> list{{13, 17}}; - m.def("kw_func4", [](const std::vector<int> &entries) { - std::string ret = "{"; - for (int i : entries) - ret += std::to_string(i) + " "; - ret.back() = '}'; - return ret; - }, py::arg("myList") = list); + m.def( + "kw_func4", + [](const std::vector<int> &entries) { + std::string ret = "{"; + for (int i : entries) { + ret += std::to_string(i) + " "; + } + ret.back() = '}'; + return ret; + }, + py::arg("myList") = list); - m.def("kw_func_udl", kw_func, "x"_a, "y"_a=300); - m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a=0); + m.def("kw_func_udl", kw_func, "x"_a, "y"_a = 300); + m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a = 0); // test_args_and_kwargs m.def("args_function", [](py::args args) -> py::tuple { - return std::move(args); + PYBIND11_WARNING_PUSH + +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") +#endif + return args; + PYBIND11_WARNING_POP }); - m.def("args_kwargs_function", [](py::args args, py::kwargs kwargs) { + m.def("args_kwargs_function", [](const py::args &args, const py::kwargs &kwargs) { return py::make_tuple(args, kwargs); }); // test_mixed_args_and_kwargs - m.def("mixed_plus_args", [](int i, double j, py::args args) { - return py::make_tuple(i, j, args); - }); - m.def("mixed_plus_kwargs", [](int i, double j, py::kwargs kwargs) { - return py::make_tuple(i, j, kwargs); - }); - auto mixed_plus_both = [](int i, double j, py::args args, py::kwargs kwargs) { + m.def("mixed_plus_args", + [](int i, double j, const py::args &args) { return py::make_tuple(i, j, args); }); + m.def("mixed_plus_kwargs", + [](int i, double j, const py::kwargs &kwargs) { return py::make_tuple(i, j, kwargs); }); + auto mixed_plus_both = [](int i, double j, const py::args &args, const py::kwargs &kwargs) { return py::make_tuple(i, j, args, kwargs); }; m.def("mixed_plus_args_kwargs", mixed_plus_both); - m.def("mixed_plus_args_kwargs_defaults", mixed_plus_both, - py::arg("i") = 1, py::arg("j") = 3.14159); - - // test_args_refcount - // PyPy needs a garbage collection to get the reference count values to match CPython's behaviour - #ifdef PYPY_VERSION - #define GC_IF_NEEDED ConstructorStats::gc() - #else - #define GC_IF_NEEDED - #endif - m.def("arg_refcount_h", [](py::handle h) { GC_IF_NEEDED; return h.ref_count(); }); - m.def("arg_refcount_h", [](py::handle h, py::handle, py::handle) { GC_IF_NEEDED; return h.ref_count(); }); - m.def("arg_refcount_o", [](py::object o) { GC_IF_NEEDED; return o.ref_count(); }); + m.def("mixed_plus_args_kwargs_defaults", + mixed_plus_both, + py::arg("i") = 1, + py::arg("j") = 3.14159); + + m.def( + "args_kwonly", + [](int i, double j, const py::args &args, int z) { return py::make_tuple(i, j, args, z); }, + "i"_a, + "j"_a, + "z"_a); + m.def( + "args_kwonly_kwargs", + [](int i, double j, const py::args &args, int z, const py::kwargs &kwargs) { + return py::make_tuple(i, j, args, z, kwargs); + }, + "i"_a, + "j"_a, + py::kw_only{}, + "z"_a); + m.def( + "args_kwonly_kwargs_defaults", + [](int i, double j, const py::args &args, int z, const py::kwargs &kwargs) { + return py::make_tuple(i, j, args, z, kwargs); + }, + "i"_a = 1, + "j"_a = 3.14159, + "z"_a = 42); + m.def( + "args_kwonly_full_monty", + [](int h, int i, double j, const py::args &args, int z, const py::kwargs &kwargs) { + return py::make_tuple(h, i, j, args, z, kwargs); + }, + py::arg() = 1, + py::arg() = 2, + py::pos_only{}, + "j"_a = 3.14159, + "z"_a = 42); + +// test_args_refcount +// PyPy needs a garbage collection to get the reference count values to match CPython's behaviour +#ifdef PYPY_VERSION +# define GC_IF_NEEDED ConstructorStats::gc() +#else +# define GC_IF_NEEDED +#endif + m.def("arg_refcount_h", [](py::handle h) { + GC_IF_NEEDED; + return h.ref_count(); + }); + m.def("arg_refcount_h", [](py::handle h, py::handle, py::handle) { + GC_IF_NEEDED; + return h.ref_count(); + }); + m.def("arg_refcount_o", [](const py::object &o) { + GC_IF_NEEDED; + return o.ref_count(); + }); m.def("args_refcount", [](py::args a) { GC_IF_NEEDED; py::tuple t(a.size()); - for (size_t i = 0; i < a.size(); i++) + for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: t[i] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i))); + } return t; }); - m.def("mixed_args_refcount", [](py::object o, py::args a) { + m.def("mixed_args_refcount", [](const py::object &o, py::args a) { GC_IF_NEEDED; py::tuple t(a.size() + 1); t[0] = o.ref_count(); - for (size_t i = 0; i < a.size(); i++) + for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: t[i + 1] = (int) Py_REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast<py::ssize_t>(i))); + } return t; }); // pybind11 won't allow these to be bound: args and kwargs, if present, must be at the end. // Uncomment these to test that the static_assert is indeed working: -// m.def("bad_args1", [](py::args, int) {}); -// m.def("bad_args2", [](py::kwargs, int) {}); -// m.def("bad_args3", [](py::kwargs, py::args) {}); -// m.def("bad_args4", [](py::args, int, py::kwargs) {}); -// m.def("bad_args5", [](py::args, py::kwargs, int) {}); -// m.def("bad_args6", [](py::args, py::args) {}); -// m.def("bad_args7", [](py::kwargs, py::kwargs) {}); + // m.def("bad_args1", [](py::args, int) {}); + // m.def("bad_args2", [](py::kwargs, int) {}); + // m.def("bad_args3", [](py::kwargs, py::args) {}); + // m.def("bad_args4", [](py::args, int, py::kwargs) {}); + // m.def("bad_args5", [](py::args, py::kwargs, int) {}); + // m.def("bad_args6", [](py::args, py::args) {}); + // m.def("bad_args7", [](py::kwargs, py::kwargs) {}); // test_keyword_only_args - m.def("kw_only_all", [](int i, int j) { return py::make_tuple(i, j); }, - py::kw_only(), py::arg("i"), py::arg("j")); - m.def("kw_only_some", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, - py::arg(), py::kw_only(), py::arg("j"), py::arg("k")); - m.def("kw_only_with_defaults", [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, - py::arg() = 3, "j"_a = 4, py::kw_only(), "k"_a = 5, "z"_a); - m.def("kw_only_mixed", [](int i, int j) { return py::make_tuple(i, j); }, - "i"_a, py::kw_only(), "j"_a); - m.def("kw_only_plus_more", [](int i, int j, int k, py::kwargs kwargs) { - return py::make_tuple(i, j, k, kwargs); }, - py::arg() /* positional */, py::arg("j") = -1 /* both */, py::kw_only(), py::arg("k") /* kw-only */); + m.def( + "kw_only_all", + [](int i, int j) { return py::make_tuple(i, j); }, + py::kw_only(), + py::arg("i"), + py::arg("j")); + m.def( + "kw_only_some", + [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg(), + py::kw_only(), + py::arg("j"), + py::arg("k")); + m.def( + "kw_only_with_defaults", + [](int i, int j, int k, int z) { return py::make_tuple(i, j, k, z); }, + py::arg() = 3, + "j"_a = 4, + py::kw_only(), + "k"_a = 5, + "z"_a); + m.def( + "kw_only_mixed", + [](int i, int j) { return py::make_tuple(i, j); }, + "i"_a, + py::kw_only(), + "j"_a); + m.def( + "kw_only_plus_more", + [](int i, int j, int k, const py::kwargs &kwargs) { + return py::make_tuple(i, j, k, kwargs); + }, + py::arg() /* positional */, + py::arg("j") = -1 /* both */, + py::kw_only(), + py::arg("k") /* kw-only */); m.def("register_invalid_kw_only", [](py::module_ m) { - m.def("bad_kw_only", [](int i, int j) { return py::make_tuple(i, j); }, - py::kw_only(), py::arg() /* invalid unnamed argument */, "j"_a); + m.def( + "bad_kw_only", + [](int i, int j) { return py::make_tuple(i, j); }, + py::kw_only(), + py::arg() /* invalid unnamed argument */, + "j"_a); }); // test_positional_only_args - m.def("pos_only_all", [](int i, int j) { return py::make_tuple(i, j); }, - py::arg("i"), py::arg("j"), py::pos_only()); - m.def("pos_only_mix", [](int i, int j) { return py::make_tuple(i, j); }, - py::arg("i"), py::pos_only(), py::arg("j")); - m.def("pos_kw_only_mix", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, - py::arg("i"), py::pos_only(), py::arg("j"), py::kw_only(), py::arg("k")); - m.def("pos_only_def_mix", [](int i, int j, int k) { return py::make_tuple(i, j, k); }, - py::arg("i"), py::arg("j") = 2, py::pos_only(), py::arg("k") = 3); - + m.def( + "pos_only_all", + [](int i, int j) { return py::make_tuple(i, j); }, + py::arg("i"), + py::arg("j"), + py::pos_only()); + m.def( + "pos_only_mix", + [](int i, int j) { return py::make_tuple(i, j); }, + py::arg("i"), + py::pos_only(), + py::arg("j")); + m.def( + "pos_kw_only_mix", + [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg("i"), + py::pos_only(), + py::arg("j"), + py::kw_only(), + py::arg("k")); + m.def( + "pos_only_def_mix", + [](int i, int j, int k) { return py::make_tuple(i, j, k); }, + py::arg("i"), + py::arg("j") = 2, + py::pos_only(), + py::arg("k") = 3); // These should fail to compile: +#ifdef PYBIND11_NEVER_DEFINED_EVER // argument annotations are required when using kw_only -// m.def("bad_kw_only1", [](int) {}, py::kw_only()); + m.def( + "bad_kw_only1", [](int) {}, py::kw_only()); // can't specify both `py::kw_only` and a `py::args` argument -// m.def("bad_kw_only2", [](int i, py::args) {}, py::kw_only(), "i"_a); + m.def( + "bad_kw_only2", [](int i, py::args) {}, py::kw_only(), "i"_a); +#endif // test_function_signatures (along with most of the above) - struct KWClass { void foo(int, float) {} }; + struct KWClass { + void foo(int, float) {} + }; py::class_<KWClass>(m, "KWClass") .def("foo0", &KWClass::foo) .def("foo1", &KWClass::foo, "x"_a, "y"_a); // Make sure a class (not an instance) can be used as a default argument. // The return value doesn't matter, only that the module is importable. - m.def("class_default_argument", [](py::object a) { return py::repr(a); }, + m.def( + "class_default_argument", + [](py::object a) { return py::repr(std::move(a)); }, "a"_a = py::module_::import("decimal").attr("Decimal")); + + // Initial implementation of kw_only was broken when used on a method/constructor before any + // other arguments + // https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987 + + struct first_arg_kw_only {}; + py::class_<first_arg_kw_only>(m, "first_arg_kw_only") + .def(py::init([](int) { return first_arg_kw_only(); }), + py::kw_only(), // This being before any args was broken + py::arg("i") = 0) + .def( + "method", + [](first_arg_kw_only &, int, int) {}, + py::kw_only(), // and likewise here + py::arg("i") = 1, + py::arg("j") = 2) + // Closely related: pos_only marker didn't show up properly when it was before any other + // arguments (although that is fairly useless in practice). + .def( + "pos_only", + [](first_arg_kw_only &, int, int) {}, + py::pos_only{}, + py::arg("i"), + py::arg("j")); } diff --git a/tests/test_kwargs_and_defaults.py b/tests/test_kwargs_and_defaults.py index 12fe705b..7174726f 100644 --- a/tests/test_kwargs_and_defaults.py +++ b/tests/test_kwargs_and_defaults.py @@ -1,8 +1,5 @@ -# -*- coding: utf-8 -*- import pytest -import env # noqa: F401 - from pybind11_tests import kwargs_and_defaults as m @@ -28,7 +25,7 @@ def test_function_signatures(doc): ) -def test_named_arguments(msg): +def test_named_arguments(): assert m.kw_func0(5, 10) == "x=5, y=10" assert m.kw_func1(5, 10) == "x=5, y=10" @@ -46,8 +43,7 @@ def test_named_arguments(msg): # noinspection PyArgumentList m.kw_func2(x=5, y=10, z=12) assert excinfo.match( - r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))" - + "{3}$" + r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$)){3}$" ) assert m.kw_func4() == "{13 17}" @@ -62,7 +58,7 @@ def test_arg_and_kwargs(): assert m.args_function(*args) == args args = "a1", "a2" - kwargs = dict(arg3="a3", arg4=4) + kwargs = {"arg3": "a3", "arg4": 4} assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) @@ -83,7 +79,7 @@ def test_mixed_args_and_kwargs(msg): 1. (arg0: int, arg1: float, *args) -> tuple Invoked with: 1 - """ # noqa: E501 line too long + """ ) with pytest.raises(TypeError) as excinfo: assert mpa() @@ -94,7 +90,7 @@ def test_mixed_args_and_kwargs(msg): 1. (arg0: int, arg1: float, *args) -> tuple Invoked with: - """ # noqa: E501 line too long + """ ) assert mpk(-2, 3.5, pi=3.14159, e=2.71828) == ( @@ -128,7 +124,7 @@ def test_mixed_args_and_kwargs(msg): 1. (i: int = 1, j: float = 3.14159, *args, **kwargs) -> tuple Invoked with: 1; kwargs: i=1 - """ # noqa: E501 line too long + """ ) with pytest.raises(TypeError) as excinfo: assert mpakd(1, 2, j=1) @@ -139,9 +135,56 @@ def test_mixed_args_and_kwargs(msg): 1. (i: int = 1, j: float = 3.14159, *args, **kwargs) -> tuple Invoked with: 1, 2; kwargs: j=1 - """ # noqa: E501 line too long + """ + ) + + # Arguments after a py::args are automatically keyword-only (pybind 2.9+) + assert m.args_kwonly(2, 2.5, z=22) == (2, 2.5, (), 22) + assert m.args_kwonly(2, 2.5, "a", "b", "c", z=22) == (2, 2.5, ("a", "b", "c"), 22) + assert m.args_kwonly(z=22, i=4, j=16) == (4, 16, (), 22) + + with pytest.raises(TypeError) as excinfo: + assert m.args_kwonly(2, 2.5, 22) # missing z= keyword + assert ( + msg(excinfo.value) + == """ + args_kwonly(): incompatible function arguments. The following argument types are supported: + 1. (i: int, j: float, *args, z: int) -> tuple + + Invoked with: 2, 2.5, 22 + """ ) + assert m.args_kwonly_kwargs(i=1, k=4, j=10, z=-1, y=9) == ( + 1, + 10, + (), + -1, + {"k": 4, "y": 9}, + ) + assert m.args_kwonly_kwargs(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, z=11, y=12) == ( + 1, + 2, + (3, 4, 5, 6, 7, 8, 9, 10), + 11, + {"y": 12}, + ) + assert ( + m.args_kwonly_kwargs.__doc__ + == "args_kwonly_kwargs(i: int, j: float, *args, z: int, **kwargs) -> tuple\n" + ) + + assert ( + m.args_kwonly_kwargs_defaults.__doc__ + == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" + ) + assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {}) + assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {}) + assert m.args_kwonly_kwargs_defaults(z=-99) == (1, 3.14159, (), -99, {}) + assert m.args_kwonly_kwargs_defaults(5, 6, 7, 8) == (5, 6, (7, 8), 42, {}) + assert m.args_kwonly_kwargs_defaults(5, 6, 7, m=8) == (5, 6, (7,), 42, {"m": 8}) + assert m.args_kwonly_kwargs_defaults(5, 6, 7, m=8, z=9) == (5, 6, (7,), 9, {"m": 8}) + def test_keyword_only_args(msg): assert m.kw_only_all(i=1, j=2) == (1, 2) @@ -179,12 +222,25 @@ def test_keyword_only_args(msg): assert ( msg(excinfo.value) == """ - arg(): cannot specify an unnamed argument after an kw_only() annotation + arg(): cannot specify an unnamed argument after a kw_only() annotation or args() argument """ ) + # https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987 + x = m.first_arg_kw_only(i=1) + x.method() + x.method(i=1, j=2) + assert ( + m.first_arg_kw_only.__init__.__doc__ + == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" + ) + assert ( + m.first_arg_kw_only.method.__doc__ + == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" + ) + -def test_positional_only_args(msg): +def test_positional_only_args(): assert m.pos_only_all(1, 2) == (1, 2) assert m.pos_only_all(2, 1) == (2, 1) @@ -223,19 +279,67 @@ def test_positional_only_args(msg): m.pos_only_def_mix(1, j=4) assert "incompatible function arguments" in str(excinfo.value) + # Mix it with args and kwargs: + assert ( + m.args_kwonly_full_monty.__doc__ + == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" + ) + assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {}) + assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {}) + assert m.args_kwonly_full_monty(8, 9) == (8, 9, 3.14159, (), 42, {}) + assert m.args_kwonly_full_monty(8, 9, 10) == (8, 9, 10.0, (), 42, {}) + assert m.args_kwonly_full_monty(3, 4, 5, 6, 7, m=8, z=9) == ( + 3, + 4, + 5.0, + ( + 6, + 7, + ), + 9, + {"m": 8}, + ) + assert m.args_kwonly_full_monty(3, 4, 5, 6, 7, m=8, z=9) == ( + 3, + 4, + 5.0, + ( + 6, + 7, + ), + 9, + {"m": 8}, + ) + assert m.args_kwonly_full_monty(5, j=7, m=8, z=9) == (5, 2, 7.0, (), 9, {"m": 8}) + assert m.args_kwonly_full_monty(i=5, j=7, m=8, z=9) == ( + 1, + 2, + 7.0, + (), + 9, + {"i": 5, "m": 8}, + ) + + # pos_only at the beginning of the argument list was "broken" in how it was displayed (though + # this is fairly useless in practice). Related to: + # https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987 + assert ( + m.first_arg_kw_only.pos_only.__doc__ + == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" + ) + def test_signatures(): - assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__ - assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__ - assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__ - assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__ + assert m.kw_only_all.__doc__ == "kw_only_all(*, i: int, j: int) -> tuple\n" + assert m.kw_only_mixed.__doc__ == "kw_only_mixed(i: int, *, j: int) -> tuple\n" + assert m.pos_only_all.__doc__ == "pos_only_all(i: int, j: int, /) -> tuple\n" + assert m.pos_only_mix.__doc__ == "pos_only_mix(i: int, /, j: int) -> tuple\n" assert ( - "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" - == m.pos_kw_only_mix.__doc__ + m.pos_kw_only_mix.__doc__ + == "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" ) -@pytest.mark.xfail("env.PYPY and env.PY2", reason="PyPy2 doesn't double count") def test_args_refcount(): """Issue/PR #1216 - py::args elements get double-inc_ref()ed when combined with regular arguments""" diff --git a/tests/test_local_bindings.cpp b/tests/test_local_bindings.cpp index c61e3888..13736774 100644 --- a/tests/test_local_bindings.cpp +++ b/tests/test_local_bindings.cpp @@ -8,11 +8,14 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "local_bindings.h" #include <pybind11/stl.h> #include <pybind11/stl_bind.h> + +#include "local_bindings.h" +#include "pybind11_tests.h" + #include <numeric> +#include <utility> TEST_SUBMODULE(local_bindings, m) { // test_load_external @@ -21,9 +24,9 @@ TEST_SUBMODULE(local_bindings, m) { // test_local_bindings // Register a class with py::module_local: - bind_local<LocalType, -1>(m, "LocalType", py::module_local()) - .def("get3", [](LocalType &t) { return t.i + 3; }) - ; + bind_local<LocalType, -1>(m, "LocalType", py::module_local()).def("get3", [](LocalType &t) { + return t.i + 3; + }); m.def("local_value", [](LocalType &l) { return l.i; }); @@ -32,20 +35,21 @@ TEST_SUBMODULE(local_bindings, m) { // one, in pybind11_cross_module_tests.cpp, is designed to fail): bind_local<NonLocalType, 0>(m, "NonLocalType") .def(py::init<int>()) - .def("get", [](LocalType &i) { return i.i; }) - ; + .def("get", [](LocalType &i) { return i.i; }); // test_duplicate_local - // py::module_local declarations should be visible across compilation units that get linked together; - // this tries to register a duplicate local. It depends on a definition in test_class.cpp and - // should raise a runtime error from the duplicate definition attempt. If test_class isn't - // available it *also* throws a runtime error (with "test_class not enabled" as value). + // py::module_local declarations should be visible across compilation units that get linked + // together; this tries to register a duplicate local. It depends on a definition in + // test_class.cpp and should raise a runtime error from the duplicate definition attempt. If + // test_class isn't available it *also* throws a runtime error (with "test_class not enabled" + // as value). m.def("register_local_external", [m]() { auto main = py::module_::import("pybind11_tests"); if (py::hasattr(main, "class_")) { bind_local<LocalExternal, 7>(m, "LocalExternal", py::module_local()); + } else { + throw std::runtime_error("test_class not enabled"); } - else throw std::runtime_error("test_class not enabled"); }); // test_stl_bind_local @@ -75,23 +79,24 @@ TEST_SUBMODULE(local_bindings, m) { m.def("get_mixed_lg", [](int i) { return MixedLocalGlobal(i); }); // test_internal_locals_differ - m.def("local_cpp_types_addr", []() { return (uintptr_t) &py::detail::registered_local_types_cpp(); }); + m.def("local_cpp_types_addr", + []() { return (uintptr_t) &py::detail::get_local_internals().registered_types_cpp; }); // test_stl_caster_vs_stl_bind - m.def("load_vector_via_caster", [](std::vector<int> v) { - return std::accumulate(v.begin(), v.end(), 0); - }); + m.def("load_vector_via_caster", + [](std::vector<int> v) { return std::accumulate(v.begin(), v.end(), 0); }); // test_cross_module_calls m.def("return_self", [](LocalVec *v) { return v; }); m.def("return_copy", [](const LocalVec &v) { return LocalVec(v); }); - class Cat : public pets::Pet { public: Cat(std::string name) : Pet(name) {}; }; - py::class_<pets::Pet>(m, "Pet", py::module_local()) - .def("get_name", &pets::Pet::name); + class Cat : public pets::Pet { + public: + explicit Cat(std::string name) : Pet(std::move(name)) {} + }; + py::class_<pets::Pet>(m, "Pet", py::module_local()).def("get_name", &pets::Pet::name); // Binding for local extending class: - py::class_<Cat, pets::Pet>(m, "Cat") - .def(py::init<std::string>()); + py::class_<Cat, pets::Pet>(m, "Cat").def(py::init<std::string>()); m.def("pet_name", [](pets::Pet &p) { return p.name(); }); py::class_<MixGL>(m, "MixGL").def(py::init<int>()); diff --git a/tests/test_local_bindings.py b/tests/test_local_bindings.py index a38564b6..d6418773 100644 --- a/tests/test_local_bindings.py +++ b/tests/test_local_bindings.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- import pytest import env # noqa: F401 - from pybind11_tests import local_bindings as m @@ -132,7 +130,8 @@ def test_stl_bind_global(): def test_mixed_local_global(): """Local types take precedence over globally registered types: a module with a `module_local` type can be registered even if the type is already registered globally. With the module, - casting will go to the local type; outside the module casting goes to the global type.""" + casting will go to the local type; outside the module casting goes to the global type. + """ import pybind11_cross_module_tests as cm m.register_mixed_global() @@ -201,7 +200,7 @@ def test_stl_caster_vs_stl_bind(msg): 1. (arg0: pybind11_cross_module_tests.VectorInt) -> int Invoked with: [1, 2, 3] - """ # noqa: E501 line too long + """ ) diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index f99909bd..31d46eb7 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -8,8 +8,8 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include "constructor_stats.h" +#include "pybind11_tests.h" #if !defined(PYBIND11_OVERLOAD_CAST) template <typename... Args> @@ -19,55 +19,61 @@ using overload_cast_ = pybind11::detail::overload_cast_impl<Args...>; class ExampleMandA { public: ExampleMandA() { print_default_created(this); } - ExampleMandA(int value) : value(value) { print_created(this, value); } + explicit ExampleMandA(int value) : value(value) { print_created(this, value); } ExampleMandA(const ExampleMandA &e) : value(e.value) { print_copy_created(this); } - ExampleMandA(std::string&&) {} - ExampleMandA(ExampleMandA &&e) : value(e.value) { print_move_created(this); } + explicit ExampleMandA(std::string &&) {} + ExampleMandA(ExampleMandA &&e) noexcept : value(e.value) { print_move_created(this); } ~ExampleMandA() { print_destroyed(this); } - std::string toString() { - return "ExampleMandA[value=" + std::to_string(value) + "]"; - } + std::string toString() const { return "ExampleMandA[value=" + std::to_string(value) + "]"; } - void operator=(const ExampleMandA &e) { print_copy_assigned(this); value = e.value; } - void operator=(ExampleMandA &&e) { print_move_assigned(this); value = e.value; } + void operator=(const ExampleMandA &e) { + print_copy_assigned(this); + value = e.value; + } + void operator=(ExampleMandA &&e) noexcept { + print_move_assigned(this); + value = e.value; + } + // NOLINTNEXTLINE(performance-unnecessary-value-param) void add1(ExampleMandA other) { value += other.value; } // passing by value void add2(ExampleMandA &other) { value += other.value; } // passing by reference void add3(const ExampleMandA &other) { value += other.value; } // passing by const reference void add4(ExampleMandA *other) { value += other->value; } // passing by pointer void add5(const ExampleMandA *other) { value += other->value; } // passing by const pointer - void add6(int other) { value += other; } // passing by value - void add7(int &other) { value += other; } // passing by reference - void add8(const int &other) { value += other; } // passing by const reference - void add9(int *other) { value += *other; } // passing by pointer - void add10(const int *other) { value += *other; } // passing by const pointer - - void consume_str(std::string&&) {} - - ExampleMandA self1() { return *this; } // return by value - ExampleMandA &self2() { return *this; } // return by reference - const ExampleMandA &self3() { return *this; } // return by const reference - ExampleMandA *self4() { return this; } // return by pointer - const ExampleMandA *self5() { return this; } // return by const pointer - - int internal1() { return value; } // return by value - int &internal2() { return value; } // return by reference - const int &internal3() { return value; } // return by const reference - int *internal4() { return &value; } // return by pointer - const int *internal5() { return &value; } // return by const pointer - - py::str overloaded() { return "()"; } - py::str overloaded(int) { return "(int)"; } - py::str overloaded(int, float) { return "(int, float)"; } - py::str overloaded(float, int) { return "(float, int)"; } - py::str overloaded(int, int) { return "(int, int)"; } + void add6(int other) { value += other; } // passing by value + void add7(int &other) { value += other; } // passing by reference + void add8(const int &other) { value += other; } // passing by const reference + // NOLINTNEXTLINE(readability-non-const-parameter) Deliberately non-const for testing + void add9(int *other) { value += *other; } // passing by pointer + void add10(const int *other) { value += *other; } // passing by const pointer + + void consume_str(std::string &&) {} + + ExampleMandA self1() { return *this; } // return by value + ExampleMandA &self2() { return *this; } // return by reference + const ExampleMandA &self3() const { return *this; } // return by const reference + ExampleMandA *self4() { return this; } // return by pointer + const ExampleMandA *self5() const { return this; } // return by const pointer + + int internal1() const { return value; } // return by value + int &internal2() { return value; } // return by reference + const int &internal3() const { return value; } // return by const reference + int *internal4() { return &value; } // return by pointer + const int *internal5() { return &value; } // return by const pointer + + py::str overloaded() { return "()"; } + py::str overloaded(int) { return "(int)"; } + py::str overloaded(int, float) { return "(int, float)"; } + py::str overloaded(float, int) { return "(float, int)"; } + py::str overloaded(int, int) { return "(int, int)"; } py::str overloaded(float, float) { return "(float, float)"; } - py::str overloaded(int) const { return "(int) const"; } - py::str overloaded(int, float) const { return "(int, float) const"; } - py::str overloaded(float, int) const { return "(float, int) const"; } - py::str overloaded(int, int) const { return "(int, int) const"; } + py::str overloaded(int) const { return "(int) const"; } + py::str overloaded(int, float) const { return "(int, float) const"; } + py::str overloaded(float, int) const { return "(float, int) const"; } + py::str overloaded(int, int) const { return "(int, int) const"; } py::str overloaded(float, float) const { return "(float, float) const"; } static py::str overloaded(float) { return "static float"; } @@ -109,25 +115,40 @@ UserType TestPropRVP::sv1(1); UserType TestPropRVP::sv2(1); // Test None-allowed py::arg argument policy -class NoneTester { public: int answer = 42; }; +class NoneTester { +public: + int answer = 42; +}; int none1(const NoneTester &obj) { return obj.answer; } int none2(NoneTester *obj) { return obj ? obj->answer : -1; } int none3(std::shared_ptr<NoneTester> &obj) { return obj ? obj->answer : -1; } int none4(std::shared_ptr<NoneTester> *obj) { return obj && *obj ? (*obj)->answer : -1; } -int none5(std::shared_ptr<NoneTester> obj) { return obj ? obj->answer : -1; } +int none5(const std::shared_ptr<NoneTester> &obj) { return obj ? obj->answer : -1; } + +// Issue #2778: implicit casting from None to object (not pointer) +class NoneCastTester { +public: + int answer = -1; + NoneCastTester() = default; + explicit NoneCastTester(int v) : answer(v) {} +}; struct StrIssue { int val = -1; StrIssue() = default; - StrIssue(int i) : val{i} {} + explicit StrIssue(int i) : val{i} {} }; -// Issues #854, #910: incompatible function args when member function/pointer is in unregistered base class +// Issues #854, #910: incompatible function args when member function/pointer is in unregistered +// base class class UnregisteredBase { public: void do_nothing() const {} - void increase_value() { rw_value++; ro_value += 0.25; } + void increase_value() { + rw_value++; + ro_value += 0.25; + } void set_int(int v) { rw_value = v; } int get_int() const { return rw_value; } double get_double() const { return ro_value; } @@ -148,13 +169,53 @@ struct RefQualified { int constRefQualified(int other) const & { return value + other; } }; +// Test rvalue ref param +struct RValueRefParam { + std::size_t func1(std::string &&s) { return s.size(); } + std::size_t func2(std::string &&s) const { return s.size(); } + std::size_t func3(std::string &&s) & { return s.size(); } + std::size_t func4(std::string &&s) const & { return s.size(); } +}; + +namespace pybind11_tests { +namespace exercise_is_setter { + +struct FieldBase { + int int_value() const { return int_value_; } + + FieldBase &SetIntValue(int int_value) { + int_value_ = int_value; + return *this; + } + +private: + int int_value_ = -99; +}; + +struct Field : FieldBase {}; + +void add_bindings(py::module &m) { + py::module sm = m.def_submodule("exercise_is_setter"); + // NOTE: FieldBase is not wrapped, therefore ... + py::class_<Field>(sm, "Field") + .def(py::init<>()) + .def_property( + "int_value", + &Field::int_value, + &Field::SetIntValue // ... the `FieldBase &` return value here cannot be converted. + ); +} + +} // namespace exercise_is_setter +} // namespace pybind11_tests + TEST_SUBMODULE(methods_and_attributes, m) { // test_methods_and_attributes py::class_<ExampleMandA> emna(m, "ExampleMandA"); emna.def(py::init<>()) .def(py::init<int>()) - .def(py::init<std::string&&>()) - .def(py::init<const ExampleMandA&>()) + .def(py::init<std::string &&>()) + .def(py::init<const ExampleMandA &>()) .def("add1", &ExampleMandA::add1) .def("add2", &ExampleMandA::add2) .def("add3", &ExampleMandA::add3) @@ -179,16 +240,20 @@ TEST_SUBMODULE(methods_and_attributes, m) { #if defined(PYBIND11_OVERLOAD_CAST) .def("overloaded", py::overload_cast<>(&ExampleMandA::overloaded)) .def("overloaded", py::overload_cast<int>(&ExampleMandA::overloaded)) - .def("overloaded", py::overload_cast<int, float>(&ExampleMandA::overloaded)) - .def("overloaded", py::overload_cast<float, int>(&ExampleMandA::overloaded)) - .def("overloaded", py::overload_cast<int, int>(&ExampleMandA::overloaded)) + .def("overloaded", py::overload_cast<int, float>(&ExampleMandA::overloaded)) + .def("overloaded", py::overload_cast<float, int>(&ExampleMandA::overloaded)) + .def("overloaded", py::overload_cast<int, int>(&ExampleMandA::overloaded)) .def("overloaded", py::overload_cast<float, float>(&ExampleMandA::overloaded)) .def("overloaded_float", py::overload_cast<float, float>(&ExampleMandA::overloaded)) - .def("overloaded_const", py::overload_cast<int >(&ExampleMandA::overloaded, py::const_)) - .def("overloaded_const", py::overload_cast<int, float>(&ExampleMandA::overloaded, py::const_)) - .def("overloaded_const", py::overload_cast<float, int>(&ExampleMandA::overloaded, py::const_)) - .def("overloaded_const", py::overload_cast<int, int>(&ExampleMandA::overloaded, py::const_)) - .def("overloaded_const", py::overload_cast<float, float>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", py::overload_cast<int>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", + py::overload_cast<int, float>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", + py::overload_cast<float, int>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", + py::overload_cast<int, int>(&ExampleMandA::overloaded, py::const_)) + .def("overloaded_const", + py::overload_cast<float, float>(&ExampleMandA::overloaded, py::const_)) #else // Use both the traditional static_cast method and the C++11 compatible overload_cast_ .def("overloaded", overload_cast_<>()(&ExampleMandA::overloaded)) @@ -206,16 +271,29 @@ TEST_SUBMODULE(methods_and_attributes, m) { #endif // test_no_mixed_overloads // Raise error if trying to mix static/non-static overloads on the same name: - .def_static("add_mixed_overloads1", []() { - auto emna = py::reinterpret_borrow<py::class_<ExampleMandA>>(py::module_::import("pybind11_tests.methods_and_attributes").attr("ExampleMandA")); - emna.def ("overload_mixed1", static_cast<py::str (ExampleMandA::*)(int, int)>(&ExampleMandA::overloaded)) - .def_static("overload_mixed1", static_cast<py::str ( *)(float )>(&ExampleMandA::overloaded)); - }) - .def_static("add_mixed_overloads2", []() { - auto emna = py::reinterpret_borrow<py::class_<ExampleMandA>>(py::module_::import("pybind11_tests.methods_and_attributes").attr("ExampleMandA")); - emna.def_static("overload_mixed2", static_cast<py::str ( *)(float )>(&ExampleMandA::overloaded)) - .def ("overload_mixed2", static_cast<py::str (ExampleMandA::*)(int, int)>(&ExampleMandA::overloaded)); - }) + .def_static("add_mixed_overloads1", + []() { + auto emna = py::reinterpret_borrow<py::class_<ExampleMandA>>( + py::module_::import("pybind11_tests.methods_and_attributes") + .attr("ExampleMandA")); + emna.def("overload_mixed1", + static_cast<py::str (ExampleMandA::*)(int, int)>( + &ExampleMandA::overloaded)) + .def_static( + "overload_mixed1", + static_cast<py::str (*)(float)>(&ExampleMandA::overloaded)); + }) + .def_static("add_mixed_overloads2", + []() { + auto emna = py::reinterpret_borrow<py::class_<ExampleMandA>>( + py::module_::import("pybind11_tests.methods_and_attributes") + .attr("ExampleMandA")); + emna.def_static("overload_mixed2", + static_cast<py::str (*)(float)>(&ExampleMandA::overloaded)) + .def("overload_mixed2", + static_cast<py::str (ExampleMandA::*)(int, int)>( + &ExampleMandA::overloaded)); + }) .def("__str__", &ExampleMandA::toString) .def_readwrite("value", &ExampleMandA::value); @@ -228,36 +306,41 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def(py::init<>()) .def_readonly("def_readonly", &TestProperties::value) .def_readwrite("def_readwrite", &TestProperties::value) - .def_property("def_writeonly", nullptr, - [](TestProperties& s,int v) { s.value = v; } ) + .def_property("def_writeonly", nullptr, [](TestProperties &s, int v) { s.value = v; }) .def_property("def_property_writeonly", nullptr, &TestProperties::set) .def_property_readonly("def_property_readonly", &TestProperties::get) .def_property("def_property", &TestProperties::get, &TestProperties::set) .def_property("def_property_impossible", nullptr, nullptr) .def_readonly_static("def_readonly_static", &TestProperties::static_value) .def_readwrite_static("def_readwrite_static", &TestProperties::static_value) - .def_property_static("def_writeonly_static", nullptr, - [](py::object, int v) { TestProperties::static_value = v; }) - .def_property_readonly_static("def_property_readonly_static", - [](py::object) { return TestProperties::static_get(); }) - .def_property_static("def_property_writeonly_static", nullptr, - [](py::object, int v) { return TestProperties::static_set(v); }) - .def_property_static("def_property_static", - [](py::object) { return TestProperties::static_get(); }, - [](py::object, int v) { TestProperties::static_set(v); }) - .def_property_static("static_cls", - [](py::object cls) { return cls; }, - [](py::object cls, py::function f) { f(cls); }); + .def_property_static("def_writeonly_static", + nullptr, + [](const py::object &, int v) { TestProperties::static_value = v; }) + .def_property_readonly_static( + "def_property_readonly_static", + [](const py::object &) { return TestProperties::static_get(); }) + .def_property_static( + "def_property_writeonly_static", + nullptr, + [](const py::object &, int v) { return TestProperties::static_set(v); }) + .def_property_static( + "def_property_static", + [](const py::object &) { return TestProperties::static_get(); }, + [](const py::object &, int v) { TestProperties::static_set(v); }) + .def_property_static( + "static_cls", + [](py::object cls) { return cls; }, + [](const py::object &cls, const py::function &f) { f(cls); }); py::class_<TestPropertiesOverride, TestProperties>(m, "TestPropertiesOverride") .def(py::init<>()) .def_readonly("def_readonly", &TestPropertiesOverride::value) .def_readonly_static("def_readonly_static", &TestPropertiesOverride::static_value); - auto static_get1 = [](py::object) -> const UserType & { return TestPropRVP::sv1; }; - auto static_get2 = [](py::object) -> const UserType & { return TestPropRVP::sv2; }; - auto static_set1 = [](py::object, int v) { TestPropRVP::sv1.set(v); }; - auto static_set2 = [](py::object, int v) { TestPropRVP::sv2.set(v); }; + auto static_get1 = [](const py::object &) -> const UserType & { return TestPropRVP::sv1; }; + auto static_get2 = [](const py::object &) -> const UserType & { return TestPropRVP::sv2; }; + auto static_set1 = [](const py::object &, int v) { TestPropRVP::sv1.set(v); }; + auto static_set2 = [](const py::object &, int v) { TestPropRVP::sv2.set(v); }; auto rvp_copy = py::return_value_policy::copy; // test_property_return_value_policies @@ -268,65 +351,74 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def_property_readonly("ro_func", py::cpp_function(&TestPropRVP::get2, rvp_copy)) .def_property("rw_ref", &TestPropRVP::get1, &TestPropRVP::set1) .def_property("rw_copy", &TestPropRVP::get2, &TestPropRVP::set2, rvp_copy) - .def_property("rw_func", py::cpp_function(&TestPropRVP::get2, rvp_copy), &TestPropRVP::set2) + .def_property( + "rw_func", py::cpp_function(&TestPropRVP::get2, rvp_copy), &TestPropRVP::set2) .def_property_readonly_static("static_ro_ref", static_get1) .def_property_readonly_static("static_ro_copy", static_get2, rvp_copy) .def_property_readonly_static("static_ro_func", py::cpp_function(static_get2, rvp_copy)) .def_property_static("static_rw_ref", static_get1, static_set1) .def_property_static("static_rw_copy", static_get2, static_set2, rvp_copy) - .def_property_static("static_rw_func", py::cpp_function(static_get2, rvp_copy), static_set2) + .def_property_static( + "static_rw_func", py::cpp_function(static_get2, rvp_copy), static_set2) // test_property_rvalue_policy .def_property_readonly("rvalue", &TestPropRVP::get_rvalue) - .def_property_readonly_static("static_rvalue", [](py::object) { return UserType(1); }); + .def_property_readonly_static("static_rvalue", + [](const py::object &) { return UserType(1); }); // test_metaclass_override - struct MetaclassOverride { }; + struct MetaclassOverride {}; py::class_<MetaclassOverride>(m, "MetaclassOverride", py::metaclass((PyObject *) &PyType_Type)) - .def_property_readonly_static("readonly", [](py::object) { return 1; }); + .def_property_readonly_static("readonly", [](const py::object &) { return 1; }); // test_overload_ordering - m.def("overload_order", [](std::string) { return 1; }); - m.def("overload_order", [](std::string) { return 2; }); + m.def("overload_order", [](const std::string &) { return 1; }); + m.def("overload_order", [](const std::string &) { return 2; }); m.def("overload_order", [](int) { return 3; }); - m.def("overload_order", [](int) { return 4; }, py::prepend{}); + m.def( + "overload_order", [](int) { return 4; }, py::prepend{}); #if !defined(PYPY_VERSION) // test_dynamic_attributes class DynamicClass { public: DynamicClass() { print_default_created(this); } - DynamicClass(const DynamicClass&) = delete; + DynamicClass(const DynamicClass &) = delete; ~DynamicClass() { print_destroyed(this); } }; - py::class_<DynamicClass>(m, "DynamicClass", py::dynamic_attr()) - .def(py::init()); + py::class_<DynamicClass>(m, "DynamicClass", py::dynamic_attr()).def(py::init()); - class CppDerivedDynamicClass : public DynamicClass { }; - py::class_<CppDerivedDynamicClass, DynamicClass>(m, "CppDerivedDynamicClass") - .def(py::init()); + class CppDerivedDynamicClass : public DynamicClass {}; + py::class_<CppDerivedDynamicClass, DynamicClass>(m, "CppDerivedDynamicClass").def(py::init()); #endif // test_bad_arg_default // Issue/PR #648: bad arg default debugging output -#if !defined(NDEBUG) - m.attr("debug_enabled") = true; +#if defined(PYBIND11_DETAILED_ERROR_MESSAGES) + m.attr("detailed_error_messages_enabled") = true; #else - m.attr("debug_enabled") = false; + m.attr("detailed_error_messages_enabled") = false; #endif - m.def("bad_arg_def_named", []{ + m.def("bad_arg_def_named", [] { auto m = py::module_::import("pybind11_tests"); - m.def("should_fail", [](int, UnregisteredType) {}, py::arg(), py::arg("a") = UnregisteredType()); + m.def( + "should_fail", + [](int, UnregisteredType) {}, + py::arg(), + py::arg("a") = UnregisteredType()); }); - m.def("bad_arg_def_unnamed", []{ + m.def("bad_arg_def_unnamed", [] { auto m = py::module_::import("pybind11_tests"); - m.def("should_fail", [](int, UnregisteredType) {}, py::arg(), py::arg() = UnregisteredType()); + m.def( + "should_fail", + [](int, UnregisteredType) {}, + py::arg(), + py::arg() = UnregisteredType()); }); // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. // test_accepts_none - py::class_<NoneTester, std::shared_ptr<NoneTester>>(m, "NoneTester") - .def(py::init<>()); + py::class_<NoneTester, std::shared_ptr<NoneTester>>(m, "NoneTester").def(py::init<>()); m.def("no_none1", &none1, py::arg{}.none(false)); m.def("no_none2", &none2, py::arg{}.none(false)); m.def("no_none3", &none3, py::arg{}.none(false)); @@ -341,14 +433,22 @@ TEST_SUBMODULE(methods_and_attributes, m) { m.def("no_none_kwarg", &none2, "a"_a.none(false)); m.def("no_none_kwarg_kw_only", &none2, py::kw_only(), "a"_a.none(false)); + // test_casts_none + // Issue #2778: implicit casting from None to object (not pointer) + py::class_<NoneCastTester>(m, "NoneCastTester") + .def(py::init<>()) + .def(py::init<int>()) + .def(py::init([](py::none const &) { return NoneCastTester{}; })); + py::implicitly_convertible<py::none, NoneCastTester>(); + m.def("ok_obj_or_none", [](NoneCastTester const &foo) { return foo.answer; }); + // test_str_issue // Issue #283: __str__ called on uninitialized instance when constructor arguments invalid py::class_<StrIssue>(m, "StrIssue") .def(py::init<int>()) .def(py::init<>()) - .def("__str__", [](const StrIssue &si) { - return "StrIssue[" + std::to_string(si.val) + "]"; } - ); + .def("__str__", + [](const StrIssue &si) { return "StrIssue[" + std::to_string(si.val) + "]"; }); // test_unregistered_base_implementations // @@ -362,16 +462,17 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def("increase_value", &RegisteredDerived::increase_value) .def_readwrite("rw_value", &RegisteredDerived::rw_value) .def_readonly("ro_value", &RegisteredDerived::ro_value) - // These should trigger a static_assert if uncommented - //.def_readwrite("fails", &UserType::value) // should trigger a static_assert if uncommented - //.def_readonly("fails", &UserType::value) // should trigger a static_assert if uncommented + // Uncommenting the next line should trigger a static_assert: + // .def_readwrite("fails", &UserType::value) + // Uncommenting the next line should trigger a static_assert: + // .def_readonly("fails", &UserType::value) .def_property("rw_value_prop", &RegisteredDerived::get_int, &RegisteredDerived::set_int) .def_property_readonly("ro_value_prop", &RegisteredDerived::get_double) // This one is in the registered class: - .def("sum", &RegisteredDerived::sum) - ; + .def("sum", &RegisteredDerived::sum); - using Adapted = decltype(py::method_adaptor<RegisteredDerived>(&RegisteredDerived::do_nothing)); + using Adapted + = decltype(py::method_adaptor<RegisteredDerived>(&RegisteredDerived::do_nothing)); static_assert(std::is_same<Adapted, void (RegisteredDerived::*)() const>::value, ""); // test_methods_and_attributes @@ -380,4 +481,13 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def_readonly("value", &RefQualified::value) .def("refQualified", &RefQualified::refQualified) .def("constRefQualified", &RefQualified::constRefQualified); + + py::class_<RValueRefParam>(m, "RValueRefParam") + .def(py::init<>()) + .def("func1", &RValueRefParam::func1) + .def("func2", &RValueRefParam::func2) + .def("func3", &RValueRefParam::func3) + .def("func4", &RValueRefParam::func4); + + pybind11_tests::exercise_is_setter::add_bindings(m); } diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 2aaf9331..955a85f6 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -1,10 +1,20 @@ -# -*- coding: utf-8 -*- +import sys + import pytest import env # noqa: F401 - -from pybind11_tests import methods_and_attributes as m from pybind11_tests import ConstructorStats +from pybind11_tests import methods_and_attributes as m + +NO_GETTER_MSG = ( + "unreadable attribute" if sys.version_info < (3, 11) else "object has no getter" +) +NO_SETTER_MSG = ( + "can't set attribute" if sys.version_info < (3, 11) else "object has no setter" +) +NO_DELETER_MSG = ( + "can't delete attribute" if sys.version_info < (3, 11) else "object has no deleter" +) def test_methods_and_attributes(): @@ -103,33 +113,33 @@ def test_properties(): assert instance.def_property == 3 with pytest.raises(AttributeError) as excinfo: - dummy = instance.def_property_writeonly # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + dummy = instance.def_property_writeonly # unused var + assert NO_GETTER_MSG in str(excinfo.value) instance.def_property_writeonly = 4 assert instance.def_property_readonly == 4 with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_impossible # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) with pytest.raises(AttributeError) as excinfo: instance.def_property_impossible = 5 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) def test_static_properties(): assert m.TestProperties.def_readonly_static == 1 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_readonly_static = 2 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) m.TestProperties.def_readwrite_static = 2 assert m.TestProperties.def_readwrite_static == 2 with pytest.raises(AttributeError) as excinfo: - dummy = m.TestProperties.def_writeonly_static # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + dummy = m.TestProperties.def_writeonly_static # unused var + assert NO_GETTER_MSG in str(excinfo.value) m.TestProperties.def_writeonly_static = 3 assert m.TestProperties.def_readonly_static == 3 @@ -137,14 +147,14 @@ def test_static_properties(): assert m.TestProperties.def_property_readonly_static == 3 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_property_readonly_static = 99 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) m.TestProperties.def_property_static = 4 assert m.TestProperties.def_property_static == 4 with pytest.raises(AttributeError) as excinfo: dummy = m.TestProperties.def_property_writeonly_static - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) m.TestProperties.def_property_writeonly_static = 5 assert m.TestProperties.def_property_static == 5 @@ -162,7 +172,7 @@ def test_static_properties(): with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_writeonly_static # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) instance.def_property_writeonly_static = 4 assert instance.def_property_static == 4 @@ -173,16 +183,16 @@ def test_static_properties(): # Only static attributes can be deleted del m.TestPropertiesOverride.def_readonly_static + assert hasattr(m.TestPropertiesOverride, "def_readonly_static") assert ( - hasattr(m.TestPropertiesOverride, "def_readonly_static") - and m.TestPropertiesOverride.def_readonly_static + m.TestPropertiesOverride.def_readonly_static is m.TestProperties.def_readonly_static ) assert "def_readonly_static" not in m.TestPropertiesOverride.__dict__ properties_override = m.TestPropertiesOverride() with pytest.raises(AttributeError) as excinfo: del properties_override.def_readonly - assert "can't delete attribute" in str(excinfo.value) + assert NO_DELETER_MSG in str(excinfo.value) def test_static_cls(): @@ -218,15 +228,15 @@ def test_metaclass_override(): def test_no_mixed_overloads(): - from pybind11_tests import debug_enabled + from pybind11_tests import detailed_error_messages_enabled with pytest.raises(RuntimeError) as excinfo: m.ExampleMandA.add_mixed_overloads1() assert str( excinfo.value ) == "overloading a method with both static and instance methods is not supported; " + ( - "compile in debug mode for more details" - if not debug_enabled + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more details" + if not detailed_error_messages_enabled else "error while attempting to bind static method ExampleMandA.overload_mixed1" "(arg0: float) -> str" ) @@ -236,8 +246,8 @@ def test_no_mixed_overloads(): assert str( excinfo.value ) == "overloading a method with both static and instance methods is not supported; " + ( - "compile in debug mode for more details" - if not debug_enabled + "#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more details" + if not detailed_error_messages_enabled else "error while attempting to bind instance method ExampleMandA.overload_mixed2" "(self: pybind11_tests.methods_and_attributes.ExampleMandA, arg0: int, arg1: int)" " -> str" @@ -246,10 +256,7 @@ def test_no_mixed_overloads(): @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) def test_property_return_value_policies(access): - if not access.startswith("static"): - obj = m.TestPropRVP() - else: - obj = m.TestPropRVP + obj = m.TestPropRVP() if not access.startswith("static") else m.TestPropRVP ref = getattr(obj, access + "_ref") assert ref.value == 1 @@ -346,16 +353,16 @@ def test_cyclic_gc(): def test_bad_arg_default(msg): - from pybind11_tests import debug_enabled + from pybind11_tests import detailed_error_messages_enabled with pytest.raises(RuntimeError) as excinfo: m.bad_arg_def_named() assert msg(excinfo.value) == ( "arg(): could not convert default argument 'a: UnregisteredType' in function " "'should_fail' into a Python object (type not registered yet?)" - if debug_enabled + if detailed_error_messages_enabled else "arg(): could not convert default argument into a Python object (type not registered " - "yet?). Compile in debug mode for more information." + "yet?). #define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more information." ) with pytest.raises(RuntimeError) as excinfo: @@ -363,9 +370,9 @@ def test_bad_arg_default(msg): assert msg(excinfo.value) == ( "arg(): could not convert default argument 'UnregisteredType' in function " "'should_fail' into a Python object (type not registered yet?)" - if debug_enabled + if detailed_error_messages_enabled else "arg(): could not convert default argument into a Python object (type not registered " - "yet?). Compile in debug mode for more information." + "yet?). #define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for more information." ) @@ -431,6 +438,17 @@ def test_accepts_none(msg): assert "incompatible function arguments" in str(excinfo.value) +def test_casts_none(): + """#2778: implicit casting from None to object (not pointer)""" + a = m.NoneCastTester() + assert m.ok_obj_or_none(a) == -1 + a = m.NoneCastTester(4) + assert m.ok_obj_or_none(a) == 4 + a = m.NoneCastTester(None) + assert m.ok_obj_or_none(a) == -1 + assert m.ok_obj_or_none(None) == -1 + + def test_str_issue(msg): """#283: __str__ called on uninitialized instance when constructor arguments invalid""" @@ -484,24 +502,32 @@ def test_overload_ordering(): assert m.overload_order("string") == 1 assert m.overload_order(0) == 4 - # Different for Python 2 vs. 3 - uni_name = type(u"").__name__ - assert "1. overload_order(arg0: int) -> int" in m.overload_order.__doc__ - assert ( - "2. overload_order(arg0: {}) -> int".format(uni_name) - in m.overload_order.__doc__ - ) - assert ( - "3. overload_order(arg0: {}) -> int".format(uni_name) - in m.overload_order.__doc__ - ) + assert "2. overload_order(arg0: str) -> int" in m.overload_order.__doc__ + assert "3. overload_order(arg0: str) -> int" in m.overload_order.__doc__ assert "4. overload_order(arg0: int) -> int" in m.overload_order.__doc__ with pytest.raises(TypeError) as err: m.overload_order(1.1) assert "1. (arg0: int) -> int" in str(err.value) - assert "2. (arg0: {}) -> int".format(uni_name) in str(err.value) - assert "3. (arg0: {}) -> int".format(uni_name) in str(err.value) + assert "2. (arg0: str) -> int" in str(err.value) + assert "3. (arg0: str) -> int" in str(err.value) assert "4. (arg0: int) -> int" in str(err.value) + + +def test_rvalue_ref_param(): + r = m.RValueRefParam() + assert r.func1("123") == 3 + assert r.func2("1234") == 4 + assert r.func3("12345") == 5 + assert r.func4("123456") == 6 + + +def test_is_setter(): + fld = m.exercise_is_setter.Field() + assert fld.int_value == -99 + setter_return = fld.int_value = 100 + assert isinstance(setter_return, int) + assert setter_return == 100 + assert fld.int_value == 100 diff --git a/tests/test_modules.cpp b/tests/test_modules.cpp index 67387e80..18a7ec74 100644 --- a/tests/test_modules.cpp +++ b/tests/test_modules.cpp @@ -8,8 +8,8 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include "constructor_stats.h" +#include "pybind11_tests.h" TEST_SUBMODULE(modules, m) { // test_nested_modules @@ -20,24 +20,32 @@ TEST_SUBMODULE(modules, m) { // test_reference_internal class A { public: - A(int v) : v(v) { print_created(this, v); } + explicit A(int v) : v(v) { print_created(this, v); } ~A() { print_destroyed(this); } - A(const A&) { print_copy_created(this); } - A& operator=(const A ©) { print_copy_assigned(this); v = copy.v; return *this; } - std::string toString() { return "A[" + std::to_string(v) + "]"; } + A(const A &) { print_copy_created(this); } + A &operator=(const A ©) { + print_copy_assigned(this); + v = copy.v; + return *this; + } + std::string toString() const { return "A[" + std::to_string(v) + "]"; } + private: int v; }; - py::class_<A>(m_sub, "A") - .def(py::init<int>()) - .def("__repr__", &A::toString); + py::class_<A>(m_sub, "A").def(py::init<int>()).def("__repr__", &A::toString); class B { public: B() { print_default_created(this); } ~B() { print_destroyed(this); } - B(const B&) { print_copy_created(this); } - B& operator=(const B ©) { print_copy_assigned(this); a1 = copy.a1; a2 = copy.a2; return *this; } + B(const B &) { print_copy_created(this); } + B &operator=(const B ©) { + print_copy_assigned(this); + a1 = copy.a1; + a2 = copy.a2; + return *this; + } A &get_a1() { return a1; } A &get_a2() { return a2; } @@ -46,9 +54,16 @@ TEST_SUBMODULE(modules, m) { }; py::class_<B>(m_sub, "B") .def(py::init<>()) - .def("get_a1", &B::get_a1, "Return the internal A 1", py::return_value_policy::reference_internal) - .def("get_a2", &B::get_a2, "Return the internal A 2", py::return_value_policy::reference_internal) - .def_readwrite("a1", &B::a1) // def_readonly uses an internal reference return policy by default + .def("get_a1", + &B::get_a1, + "Return the internal A 1", + py::return_value_policy::reference_internal) + .def("get_a2", + &B::get_a2, + "Return the internal A 2", + py::return_value_policy::reference_internal) + .def_readwrite("a1", &B::a1) // def_readonly uses an internal + // reference return policy by default .def_readwrite("a2", &B::a2); // This is intentionally "py::module" to verify it still can be used in place of "py::module_" @@ -57,13 +72,14 @@ TEST_SUBMODULE(modules, m) { // test_duplicate_registration // Registering two things with the same name m.def("duplicate_registration", []() { - class Dupe1 { }; - class Dupe2 { }; - class Dupe3 { }; - class DupeException { }; + class Dupe1 {}; + class Dupe2 {}; + class Dupe3 {}; + class DupeException {}; // Go ahead and leak, until we have a non-leaking py::module_ constructor - auto dm = py::module_::create_extension_module("dummy", nullptr, new py::module_::module_def); + auto dm + = py::module_::create_extension_module("dummy", nullptr, new py::module_::module_def); auto failures = py::list(); py::class_<Dupe1>(dm, "Dupe1"); @@ -74,28 +90,36 @@ TEST_SUBMODULE(modules, m) { try { py::class_<Dupe1>(dm, "Dupe1"); failures.append("Dupe1 class"); - } catch (std::runtime_error &) {} + } catch (std::runtime_error &) { + } try { dm.def("Dupe1", []() { return Dupe1(); }); failures.append("Dupe1 function"); - } catch (std::runtime_error &) {} + } catch (std::runtime_error &) { + } try { py::class_<Dupe3>(dm, "dupe1_factory"); failures.append("dupe1_factory"); - } catch (std::runtime_error &) {} + } catch (std::runtime_error &) { + } try { py::exception<Dupe3>(dm, "Dupe2"); failures.append("Dupe2"); - } catch (std::runtime_error &) {} + } catch (std::runtime_error &) { + } try { dm.def("DupeException", []() { return 30; }); failures.append("DupeException1"); - } catch (std::runtime_error &) {} + } catch (std::runtime_error &) { + } try { py::class_<DupeException>(dm, "DupeException"); failures.append("DupeException2"); - } catch (std::runtime_error &) {} + } catch (std::runtime_error &) { + } return failures; }); + + m.def("def_submodule", [](py::module_ m, const char *name) { return m.def_submodule(name); }); } diff --git a/tests/test_modules.py b/tests/test_modules.py index 3390031a..2f6d825b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,7 +1,11 @@ -# -*- coding: utf-8 -*- +import builtins + +import pytest + +import env +from pybind11_tests import ConstructorStats from pybind11_tests import modules as m from pybind11_tests.modules import subsubmodule as ms -from pybind11_tests import ConstructorStats def test_nested_modules(): @@ -54,18 +58,19 @@ def test_reference_internal(): def test_importing(): - from pybind11_tests.modules import OD from collections import OrderedDict + from pybind11_tests.modules import OD + assert OD is OrderedDict - assert str(OD([(1, "a"), (2, "b")])) == "OrderedDict([(1, 'a'), (2, 'b')])" def test_pydoc(): """Pydoc needs to be able to provide help() for everything inside a pybind11 module""" - import pybind11_tests import pydoc + import pybind11_tests + assert pybind11_tests.__name__ == "pybind11_tests" assert pybind11_tests.__doc__ == "pybind11 test module" assert pydoc.text.docmodule(pybind11_tests) @@ -82,9 +87,30 @@ def test_builtin_key_type(): Previous versions of pybind11 would add a unicode key in python 2. """ - if hasattr(__builtins__, "keys"): - keys = __builtins__.keys() - else: # this is to make pypy happy since builtins is different there. - keys = __builtins__.__dict__.keys() - - assert {type(k) for k in keys} == {str} + assert all(type(k) == str for k in dir(builtins)) + + +@pytest.mark.xfail("env.PYPY", reason="PyModule_GetName()") +def test_def_submodule_failures(): + sm = m.def_submodule(m, b"ScratchSubModuleName") # Using bytes to show it works. + assert sm.__name__ == m.__name__ + "." + "ScratchSubModuleName" + malformed_utf8 = b"\x80" + if env.PYPY: + # It is not worth the effort finding a trigger for a failure when running with PyPy. + pytest.skip("Sufficiently exercised on platforms other than PyPy.") + else: + # Meant to trigger PyModule_GetName() failure: + sm_name_orig = sm.__name__ + sm.__name__ = malformed_utf8 + try: + # We want to assert that a bad __name__ causes some kind of failure, although we do not want to exercise + # the internals of PyModule_GetName(). Currently all supported Python versions raise SystemError. If that + # changes in future Python versions, simply add the new expected exception types here. + with pytest.raises(SystemError): + m.def_submodule(sm, b"SubSubModuleName") + finally: + # Clean up to ensure nothing gets upset by a module with an invalid __name__. + sm.__name__ = sm_name_orig # Purely precautionary. + # Meant to trigger PyImport_AddModule() failure: + with pytest.raises(UnicodeDecodeError): + m.def_submodule(sm, malformed_utf8) diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index e6720080..5916ae90 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -8,13 +8,16 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include "constructor_stats.h" +#include "pybind11_tests.h" + +namespace { // Many bases for testing that multiple inheritance from many classes (i.e. requiring extra // space for holder constructed flags) works. -template <int N> struct BaseN { - BaseN(int i) : i(i) { } +template <int N> +struct BaseN { + explicit BaseN(int i) : i(i) {} int i; }; @@ -43,103 +46,139 @@ int WithStatic2::static_value2 = 2; int VanillaStaticMix1::static_value = 12; int VanillaStaticMix2::static_value = 12; +// test_multiple_inheritance_virtbase +struct Base1a { + explicit Base1a(int i) : i(i) {} + int foo() const { return i; } + int i; +}; +struct Base2a { + explicit Base2a(int i) : i(i) {} + int bar() const { return i; } + int i; +}; +struct Base12a : Base1a, Base2a { + Base12a(int i, int j) : Base1a(i), Base2a(j) {} +}; + +// test_mi_unaligned_base +// test_mi_base_return +struct I801B1 { + int a = 1; + I801B1() = default; + I801B1(const I801B1 &) = default; + virtual ~I801B1() = default; +}; +struct I801B2 { + int b = 2; + I801B2() = default; + I801B2(const I801B2 &) = default; + virtual ~I801B2() = default; +}; +struct I801C : I801B1, I801B2 {}; +struct I801D : I801C {}; // Indirect MI + +} // namespace + TEST_SUBMODULE(multiple_inheritance, m) { + // Please do not interleave `struct` and `class` definitions with bindings code, + // but implement `struct`s and `class`es in the anonymous namespace above. + // This helps keeping the smart_holder branch in sync with master. // test_multiple_inheritance_mix1 // test_multiple_inheritance_mix2 struct Base1 { - Base1(int i) : i(i) { } - int foo() { return i; } + explicit Base1(int i) : i(i) {} + int foo() const { return i; } int i; }; py::class_<Base1> b1(m, "Base1"); - b1.def(py::init<int>()) - .def("foo", &Base1::foo); + b1.def(py::init<int>()).def("foo", &Base1::foo); struct Base2 { - Base2(int i) : i(i) { } - int bar() { return i; } + explicit Base2(int i) : i(i) {} + int bar() const { return i; } int i; }; py::class_<Base2> b2(m, "Base2"); - b2.def(py::init<int>()) - .def("bar", &Base2::bar); - + b2.def(py::init<int>()).def("bar", &Base2::bar); // test_multiple_inheritance_cpp struct Base12 : Base1, Base2 { - Base12(int i, int j) : Base1(i), Base2(j) { } + Base12(int i, int j) : Base1(i), Base2(j) {} }; struct MIType : Base12 { - MIType(int i, int j) : Base12(i, j) { } + MIType(int i, int j) : Base12(i, j) {} }; py::class_<Base12, Base1, Base2>(m, "Base12"); - py::class_<MIType, Base12>(m, "MIType") - .def(py::init<int, int>()); - + py::class_<MIType, Base12>(m, "MIType").def(py::init<int, int>()); // test_multiple_inheritance_python_many_bases - #define PYBIND11_BASEN(N) py::class_<BaseN<N>>(m, "BaseN" #N).def(py::init<int>()).def("f" #N, [](BaseN<N> &b) { return b.i + N; }) - PYBIND11_BASEN( 1); PYBIND11_BASEN( 2); PYBIND11_BASEN( 3); PYBIND11_BASEN( 4); - PYBIND11_BASEN( 5); PYBIND11_BASEN( 6); PYBIND11_BASEN( 7); PYBIND11_BASEN( 8); - PYBIND11_BASEN( 9); PYBIND11_BASEN(10); PYBIND11_BASEN(11); PYBIND11_BASEN(12); - PYBIND11_BASEN(13); PYBIND11_BASEN(14); PYBIND11_BASEN(15); PYBIND11_BASEN(16); +#define PYBIND11_BASEN(N) \ + py::class_<BaseN<(N)>>(m, "BaseN" #N).def(py::init<int>()).def("f" #N, [](BaseN<N> &b) { \ + return b.i + (N); \ + }) + PYBIND11_BASEN(1); + PYBIND11_BASEN(2); + PYBIND11_BASEN(3); + PYBIND11_BASEN(4); + PYBIND11_BASEN(5); + PYBIND11_BASEN(6); + PYBIND11_BASEN(7); + PYBIND11_BASEN(8); + PYBIND11_BASEN(9); + PYBIND11_BASEN(10); + PYBIND11_BASEN(11); + PYBIND11_BASEN(12); + PYBIND11_BASEN(13); + PYBIND11_BASEN(14); + PYBIND11_BASEN(15); + PYBIND11_BASEN(16); PYBIND11_BASEN(17); // Uncommenting this should result in a compile time failure (MI can only be specified via - // template parameters because pybind has to know the types involved; see discussion in #742 for - // details). -// struct Base12v2 : Base1, Base2 { -// Base12v2(int i, int j) : Base1(i), Base2(j) { } -// }; -// py::class_<Base12v2>(m, "Base12v2", b1, b2) -// .def(py::init<int, int>()); - + // template parameters because pybind has to know the types involved; see discussion in #742 + // for details). + // struct Base12v2 : Base1, Base2 { + // Base12v2(int i, int j) : Base1(i), Base2(j) { } + // }; + // py::class_<Base12v2>(m, "Base12v2", b1, b2) + // .def(py::init<int, int>()); // test_multiple_inheritance_virtbase // Test the case where not all base classes are specified, and where pybind11 requires the // py::multiple_inheritance flag to perform proper casting between types. - struct Base1a { - Base1a(int i) : i(i) { } - int foo() { return i; } - int i; - }; py::class_<Base1a, std::shared_ptr<Base1a>>(m, "Base1a") .def(py::init<int>()) .def("foo", &Base1a::foo); - struct Base2a { - Base2a(int i) : i(i) { } - int bar() { return i; } - int i; - }; py::class_<Base2a, std::shared_ptr<Base2a>>(m, "Base2a") .def(py::init<int>()) .def("bar", &Base2a::bar); - struct Base12a : Base1a, Base2a { - Base12a(int i, int j) : Base1a(i), Base2a(j) { } - }; - py::class_<Base12a, /* Base1 missing */ Base2a, - std::shared_ptr<Base12a>>(m, "Base12a", py::multiple_inheritance()) + py::class_<Base12a, /* Base1 missing */ Base2a, std::shared_ptr<Base12a>>( + m, "Base12a", py::multiple_inheritance()) .def(py::init<int, int>()); m.def("bar_base2a", [](Base2a *b) { return b->bar(); }); - m.def("bar_base2a_sharedptr", [](std::shared_ptr<Base2a> b) { return b->bar(); }); + m.def("bar_base2a_sharedptr", [](const std::shared_ptr<Base2a> &b) { return b->bar(); }); // test_mi_unaligned_base // test_mi_base_return // Issue #801: invalid casting to derived type with MI bases - struct I801B1 { int a = 1; I801B1() = default; I801B1(const I801B1 &) = default; virtual ~I801B1() = default; }; - struct I801B2 { int b = 2; I801B2() = default; I801B2(const I801B2 &) = default; virtual ~I801B2() = default; }; - struct I801C : I801B1, I801B2 {}; - struct I801D : I801C {}; // Indirect MI // Unregistered classes: - struct I801B3 { int c = 3; virtual ~I801B3() = default; }; + struct I801B3 { + int c = 3; + virtual ~I801B3() = default; + }; struct I801E : I801B3, I801D {}; - py::class_<I801B1, std::shared_ptr<I801B1>>(m, "I801B1").def(py::init<>()).def_readonly("a", &I801B1::a); - py::class_<I801B2, std::shared_ptr<I801B2>>(m, "I801B2").def(py::init<>()).def_readonly("b", &I801B2::b); + py::class_<I801B1, std::shared_ptr<I801B1>>(m, "I801B1") + .def(py::init<>()) + .def_readonly("a", &I801B1::a); + py::class_<I801B2, std::shared_ptr<I801B2>>(m, "I801B2") + .def(py::init<>()) + .def_readonly("b", &I801B2::b); py::class_<I801C, I801B1, I801B2, std::shared_ptr<I801C>>(m, "I801C").def(py::init<>()); py::class_<I801D, I801C, std::shared_ptr<I801D>>(m, "I801D").def(py::init<>()); @@ -164,11 +203,8 @@ TEST_SUBMODULE(multiple_inheritance, m) { m.def("i801e_c", []() -> I801C * { return new I801E(); }); m.def("i801e_b2", []() -> I801B2 * { return new I801E(); }); - // test_mi_static_properties - py::class_<Vanilla>(m, "Vanilla") - .def(py::init<>()) - .def("vanilla", &Vanilla::vanilla); + py::class_<Vanilla>(m, "Vanilla").def(py::init<>()).def("vanilla", &Vanilla::vanilla); py::class_<WithStatic1>(m, "WithStatic1") .def(py::init<>()) @@ -180,22 +216,19 @@ TEST_SUBMODULE(multiple_inheritance, m) { .def_static("static_func2", &WithStatic2::static_func2) .def_readwrite_static("static_value2", &WithStatic2::static_value2); - py::class_<VanillaStaticMix1, Vanilla, WithStatic1, WithStatic2>( - m, "VanillaStaticMix1") + py::class_<VanillaStaticMix1, Vanilla, WithStatic1, WithStatic2>(m, "VanillaStaticMix1") .def(py::init<>()) .def_static("static_func", &VanillaStaticMix1::static_func) .def_readwrite_static("static_value", &VanillaStaticMix1::static_value); - py::class_<VanillaStaticMix2, WithStatic1, Vanilla, WithStatic2>( - m, "VanillaStaticMix2") + py::class_<VanillaStaticMix2, WithStatic1, Vanilla, WithStatic2>(m, "VanillaStaticMix2") .def(py::init<>()) .def_static("static_func", &VanillaStaticMix2::static_func) .def_readwrite_static("static_value", &VanillaStaticMix2::static_value); - - struct WithDict { }; - struct VanillaDictMix1 : Vanilla, WithDict { }; - struct VanillaDictMix2 : WithDict, Vanilla { }; + struct WithDict {}; + struct VanillaDictMix1 : Vanilla, WithDict {}; + struct VanillaDictMix2 : WithDict, Vanilla {}; py::class_<WithDict>(m, "WithDict", py::dynamic_attr()).def(py::init<>()); py::class_<VanillaDictMix1, Vanilla, WithDict>(m, "VanillaDictMix1").def(py::init<>()); py::class_<VanillaDictMix2, WithDict, Vanilla>(m, "VanillaDictMix2").def(py::init<>()); @@ -203,16 +236,106 @@ TEST_SUBMODULE(multiple_inheritance, m) { // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance // All of these have int members so that there will be various unequal pointers involved. - struct B { int b; B() = default; B(const B&) = default; virtual ~B() = default; }; - struct C0 : public virtual B { int c0; }; - struct C1 : public virtual B { int c1; }; - struct D : public C0, public C1 { int d; }; - py::class_<B>(m, "B") - .def("b", [](B *self) { return self; }); - py::class_<C0, B>(m, "C0") - .def("c0", [](C0 *self) { return self; }); - py::class_<C1, B>(m, "C1") - .def("c1", [](C1 *self) { return self; }); - py::class_<D, C0, C1>(m, "D") - .def(py::init<>()); + struct B { + int b; + B() = default; + B(const B &) = default; + virtual ~B() = default; + }; + struct C0 : public virtual B { + int c0; + }; + struct C1 : public virtual B { + int c1; + }; + struct D : public C0, public C1 { + int d; + }; + py::class_<B>(m, "B").def("b", [](B *self) { return self; }); + py::class_<C0, B>(m, "C0").def("c0", [](C0 *self) { return self; }); + py::class_<C1, B>(m, "C1").def("c1", [](C1 *self) { return self; }); + py::class_<D, C0, C1>(m, "D").def(py::init<>()); + + // test_pr3635_diamond_* + // - functions are get_{base}_{var}, return {var} + struct MVB { + MVB() = default; + MVB(const MVB &) = default; + virtual ~MVB() = default; + + int b = 1; + int get_b_b() const { return b; } + }; + struct MVC : virtual MVB { + int c = 2; + int get_c_b() const { return b; } + int get_c_c() const { return c; } + }; + struct MVD0 : virtual MVC { + int d0 = 3; + int get_d0_b() const { return b; } + int get_d0_c() const { return c; } + int get_d0_d0() const { return d0; } + }; + struct MVD1 : virtual MVC { + int d1 = 4; + int get_d1_b() const { return b; } + int get_d1_c() const { return c; } + int get_d1_d1() const { return d1; } + }; + struct MVE : virtual MVD0, virtual MVD1 { + int e = 5; + int get_e_b() const { return b; } + int get_e_c() const { return c; } + int get_e_d0() const { return d0; } + int get_e_d1() const { return d1; } + int get_e_e() const { return e; } + }; + struct MVF : virtual MVE { + int f = 6; + int get_f_b() const { return b; } + int get_f_c() const { return c; } + int get_f_d0() const { return d0; } + int get_f_d1() const { return d1; } + int get_f_e() const { return e; } + int get_f_f() const { return f; } + }; + py::class_<MVB>(m, "MVB") + .def(py::init<>()) + .def("get_b_b", &MVB::get_b_b) + .def_readwrite("b", &MVB::b); + py::class_<MVC, MVB>(m, "MVC") + .def(py::init<>()) + .def("get_c_b", &MVC::get_c_b) + .def("get_c_c", &MVC::get_c_c) + .def_readwrite("c", &MVC::c); + py::class_<MVD0, MVC>(m, "MVD0") + .def(py::init<>()) + .def("get_d0_b", &MVD0::get_d0_b) + .def("get_d0_c", &MVD0::get_d0_c) + .def("get_d0_d0", &MVD0::get_d0_d0) + .def_readwrite("d0", &MVD0::d0); + py::class_<MVD1, MVC>(m, "MVD1") + .def(py::init<>()) + .def("get_d1_b", &MVD1::get_d1_b) + .def("get_d1_c", &MVD1::get_d1_c) + .def("get_d1_d1", &MVD1::get_d1_d1) + .def_readwrite("d1", &MVD1::d1); + py::class_<MVE, MVD0, MVD1>(m, "MVE") + .def(py::init<>()) + .def("get_e_b", &MVE::get_e_b) + .def("get_e_c", &MVE::get_e_c) + .def("get_e_d0", &MVE::get_e_d0) + .def("get_e_d1", &MVE::get_e_d1) + .def("get_e_e", &MVE::get_e_e) + .def_readwrite("e", &MVE::e); + py::class_<MVF, MVE>(m, "MVF") + .def(py::init<>()) + .def("get_f_b", &MVF::get_f_b) + .def("get_f_c", &MVF::get_f_c) + .def("get_f_d0", &MVF::get_f_d0) + .def("get_f_d1", &MVF::get_f_d1) + .def("get_f_e", &MVF::get_f_e) + .def("get_f_f", &MVF::get_f_f) + .def_readwrite("f", &MVF::f); } diff --git a/tests/test_multiple_inheritance.py b/tests/test_multiple_inheritance.py index e6a7f976..3a1d88d7 100644 --- a/tests/test_multiple_inheritance.py +++ b/tests/test_multiple_inheritance.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- import pytest import env # noqa: F401 - from pybind11_tests import ConstructorStats from pybind11_tests import multiple_inheritance as m @@ -14,8 +12,7 @@ def test_multiple_inheritance_cpp(): assert mt.bar() == 4 -@pytest.mark.skipif("env.PYPY and env.PY2") -@pytest.mark.xfail("env.PYPY and not env.PY2") +@pytest.mark.xfail("env.PYPY") def test_multiple_inheritance_mix1(): class Base1: def __init__(self, i): @@ -54,15 +51,14 @@ def test_multiple_inheritance_mix2(): assert mt.bar() == 4 -@pytest.mark.skipif("env.PYPY and env.PY2") -@pytest.mark.xfail("env.PYPY and not env.PY2") +@pytest.mark.xfail("env.PYPY") def test_multiple_inheritance_python(): class MI1(m.Base1, m.Base2): def __init__(self, i, j): m.Base1.__init__(self, i) m.Base2.__init__(self, j) - class B1(object): + class B1: def v(self): return 1 @@ -97,7 +93,7 @@ def test_multiple_inheritance_python(): def v(self): return 2 - class B3(object): + class B3: def v(self): return 3 @@ -359,3 +355,139 @@ def test_diamond_inheritance(): assert d is d.c0().b() assert d is d.c1().b() assert d is d.c0().c1().b().c0().b() + + +def test_pr3635_diamond_b(): + o = m.MVB() + assert o.b == 1 + + assert o.get_b_b() == 1 + + +def test_pr3635_diamond_c(): + o = m.MVC() + assert o.b == 1 + assert o.c == 2 + + assert o.get_b_b() == 1 + assert o.get_c_b() == 1 + + assert o.get_c_c() == 2 + + +def test_pr3635_diamond_d0(): + o = m.MVD0() + assert o.b == 1 + assert o.c == 2 + assert o.d0 == 3 + + assert o.get_b_b() == 1 + assert o.get_c_b() == 1 + assert o.get_d0_b() == 1 + + assert o.get_c_c() == 2 + assert o.get_d0_c() == 2 + + assert o.get_d0_d0() == 3 + + +def test_pr3635_diamond_d1(): + o = m.MVD1() + assert o.b == 1 + assert o.c == 2 + assert o.d1 == 4 + + assert o.get_b_b() == 1 + assert o.get_c_b() == 1 + assert o.get_d1_b() == 1 + + assert o.get_c_c() == 2 + assert o.get_d1_c() == 2 + + assert o.get_d1_d1() == 4 + + +def test_pr3635_diamond_e(): + o = m.MVE() + assert o.b == 1 + assert o.c == 2 + assert o.d0 == 3 + assert o.d1 == 4 + assert o.e == 5 + + assert o.get_b_b() == 1 + assert o.get_c_b() == 1 + assert o.get_d0_b() == 1 + assert o.get_d1_b() == 1 + assert o.get_e_b() == 1 + + assert o.get_c_c() == 2 + assert o.get_d0_c() == 2 + assert o.get_d1_c() == 2 + assert o.get_e_c() == 2 + + assert o.get_d0_d0() == 3 + assert o.get_e_d0() == 3 + + assert o.get_d1_d1() == 4 + assert o.get_e_d1() == 4 + + assert o.get_e_e() == 5 + + +def test_pr3635_diamond_f(): + o = m.MVF() + assert o.b == 1 + assert o.c == 2 + assert o.d0 == 3 + assert o.d1 == 4 + assert o.e == 5 + assert o.f == 6 + + assert o.get_b_b() == 1 + assert o.get_c_b() == 1 + assert o.get_d0_b() == 1 + assert o.get_d1_b() == 1 + assert o.get_e_b() == 1 + assert o.get_f_b() == 1 + + assert o.get_c_c() == 2 + assert o.get_d0_c() == 2 + assert o.get_d1_c() == 2 + assert o.get_e_c() == 2 + assert o.get_f_c() == 2 + + assert o.get_d0_d0() == 3 + assert o.get_e_d0() == 3 + assert o.get_f_d0() == 3 + + assert o.get_d1_d1() == 4 + assert o.get_e_d1() == 4 + assert o.get_f_d1() == 4 + + assert o.get_e_e() == 5 + assert o.get_f_e() == 5 + + assert o.get_f_f() == 6 + + +def test_python_inherit_from_mi(): + """Tests extending a Python class from a single inheritor of a MI class""" + + class PyMVF(m.MVF): + g = 7 + + def get_g_g(self): + return self.g + + o = PyMVF() + + assert o.b == 1 + assert o.c == 2 + assert o.d0 == 3 + assert o.d1 == 4 + assert o.e == 5 + assert o.f == 6 + assert o.g == 7 + + assert o.get_g_g() == 7 diff --git a/tests/test_numpy_array.cpp b/tests/test_numpy_array.cpp index dca7145f..8c122a86 100644 --- a/tests/test_numpy_array.cpp +++ b/tests/test_numpy_array.cpp @@ -7,12 +7,13 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" - #include <pybind11/numpy.h> #include <pybind11/stl.h> +#include "pybind11_tests.h" + #include <cstdint> +#include <utility> // Size / dtype checks. struct DtypeCheck { @@ -21,7 +22,7 @@ struct DtypeCheck { }; template <typename T> -DtypeCheck get_dtype_check(const char* name) { +DtypeCheck get_dtype_check(const char *name) { py::module_ np = py::module_::import("numpy"); DtypeCheck check{}; check.numpy = np.attr("dtype")(np.attr(name)); @@ -30,17 +31,15 @@ DtypeCheck get_dtype_check(const char* name) { } std::vector<DtypeCheck> get_concrete_dtype_checks() { - return { - // Normalization - get_dtype_check<std::int8_t>("int8"), - get_dtype_check<std::uint8_t>("uint8"), - get_dtype_check<std::int16_t>("int16"), - get_dtype_check<std::uint16_t>("uint16"), - get_dtype_check<std::int32_t>("int32"), - get_dtype_check<std::uint32_t>("uint32"), - get_dtype_check<std::int64_t>("int64"), - get_dtype_check<std::uint64_t>("uint64") - }; + return {// Normalization + get_dtype_check<std::int8_t>("int8"), + get_dtype_check<std::uint8_t>("uint8"), + get_dtype_check<std::int16_t>("int16"), + get_dtype_check<std::uint16_t>("uint16"), + get_dtype_check<std::int32_t>("int32"), + get_dtype_check<std::uint32_t>("uint32"), + get_dtype_check<std::int64_t>("int64"), + get_dtype_check<std::uint64_t>("uint64")}; } struct DtypeSizeCheck { @@ -79,43 +78,71 @@ using arr = py::array; using arr_t = py::array_t<uint16_t, 0>; static_assert(std::is_same<arr_t::value_type, uint16_t>::value, ""); -template<typename... Ix> arr data(const arr& a, Ix... index) { +template <typename... Ix> +arr data(const arr &a, Ix... index) { return arr(a.nbytes() - a.offset_at(index...), (const uint8_t *) a.data(index...)); } -template<typename... Ix> arr data_t(const arr_t& a, Ix... index) { +template <typename... Ix> +arr data_t(const arr_t &a, Ix... index) { return arr(a.size() - a.index_at(index...), a.data(index...)); } -template<typename... Ix> arr& mutate_data(arr& a, Ix... index) { - auto ptr = (uint8_t *) a.mutable_data(index...); - for (py::ssize_t i = 0; i < a.nbytes() - a.offset_at(index...); i++) +template <typename... Ix> +arr &mutate_data(arr &a, Ix... index) { + auto *ptr = (uint8_t *) a.mutable_data(index...); + for (py::ssize_t i = 0; i < a.nbytes() - a.offset_at(index...); i++) { ptr[i] = (uint8_t) (ptr[i] * 2); + } return a; } -template<typename... Ix> arr_t& mutate_data_t(arr_t& a, Ix... index) { +template <typename... Ix> +arr_t &mutate_data_t(arr_t &a, Ix... index) { auto ptr = a.mutable_data(index...); - for (py::ssize_t i = 0; i < a.size() - a.index_at(index...); i++) + for (py::ssize_t i = 0; i < a.size() - a.index_at(index...); i++) { ptr[i]++; + } + return a; +} + +template <typename... Ix> +py::ssize_t index_at(const arr &a, Ix... idx) { + return a.index_at(idx...); +} +template <typename... Ix> +py::ssize_t index_at_t(const arr_t &a, Ix... idx) { + return a.index_at(idx...); +} +template <typename... Ix> +py::ssize_t offset_at(const arr &a, Ix... idx) { + return a.offset_at(idx...); +} +template <typename... Ix> +py::ssize_t offset_at_t(const arr_t &a, Ix... idx) { + return a.offset_at(idx...); +} +template <typename... Ix> +py::ssize_t at_t(const arr_t &a, Ix... idx) { + return a.at(idx...); +} +template <typename... Ix> +arr_t &mutate_at_t(arr_t &a, Ix... idx) { + a.mutable_at(idx...)++; return a; } -template<typename... Ix> py::ssize_t index_at(const arr& a, Ix... idx) { return a.index_at(idx...); } -template<typename... Ix> py::ssize_t index_at_t(const arr_t& a, Ix... idx) { return a.index_at(idx...); } -template<typename... Ix> py::ssize_t offset_at(const arr& a, Ix... idx) { return a.offset_at(idx...); } -template<typename... Ix> py::ssize_t offset_at_t(const arr_t& a, Ix... idx) { return a.offset_at(idx...); } -template<typename... Ix> py::ssize_t at_t(const arr_t& a, Ix... idx) { return a.at(idx...); } -template<typename... Ix> arr_t& mutate_at_t(arr_t& a, Ix... idx) { a.mutable_at(idx...)++; return a; } - -#define def_index_fn(name, type) \ - sm.def(#name, [](type a) { return name(a); }); \ - sm.def(#name, [](type a, int i) { return name(a, i); }); \ - sm.def(#name, [](type a, int i, int j) { return name(a, i, j); }); \ +#define def_index_fn(name, type) \ + sm.def(#name, [](type a) { return name(a); }); \ + sm.def(#name, [](type a, int i) { return name(a, i); }); \ + sm.def(#name, [](type a, int i, int j) { return name(a, i, j); }); \ sm.def(#name, [](type a, int i, int j, int k) { return name(a, i, j, k); }); -template <typename T, typename T2> py::handle auxiliaries(T &&r, T2 &&r2) { - if (r.ndim() != 2) throw std::domain_error("error: ndim != 2"); +template <typename T, typename T2> +py::handle auxiliaries(T &&r, T2 &&r2) { + if (r.ndim() != 2) { + throw std::domain_error("error: ndim != 2"); + } py::list l; l.append(*r.data(0, 0)); l.append(*r2.mutable_data(0, 0)); @@ -133,16 +160,18 @@ template <typename T, typename T2> py::handle auxiliaries(T &&r, T2 &&r2) { static int data_i = 42; TEST_SUBMODULE(numpy_array, sm) { - try { py::module_::import("numpy"); } - catch (...) { return; } + try { + py::module_::import("numpy"); + } catch (const py::error_already_set &) { + return; + } // test_dtypes py::class_<DtypeCheck>(sm, "DtypeCheck") .def_readonly("numpy", &DtypeCheck::numpy) .def_readonly("pybind11", &DtypeCheck::pybind11) - .def("__repr__", [](const DtypeCheck& self) { - return py::str("<DtypeCheck numpy={} pybind11={}>").format( - self.numpy, self.pybind11); + .def("__repr__", [](const DtypeCheck &self) { + return py::str("<DtypeCheck numpy={} pybind11={}>").format(self.numpy, self.pybind11); }); sm.def("get_concrete_dtype_checks", &get_concrete_dtype_checks); @@ -150,41 +179,41 @@ TEST_SUBMODULE(numpy_array, sm) { .def_readonly("name", &DtypeSizeCheck::name) .def_readonly("size_cpp", &DtypeSizeCheck::size_cpp) .def_readonly("size_numpy", &DtypeSizeCheck::size_numpy) - .def("__repr__", [](const DtypeSizeCheck& self) { - return py::str("<DtypeSizeCheck name='{}' size_cpp={} size_numpy={} dtype={}>").format( - self.name, self.size_cpp, self.size_numpy, self.dtype); + .def("__repr__", [](const DtypeSizeCheck &self) { + return py::str("<DtypeSizeCheck name='{}' size_cpp={} size_numpy={} dtype={}>") + .format(self.name, self.size_cpp, self.size_numpy, self.dtype); }); sm.def("get_platform_dtype_size_checks", &get_platform_dtype_size_checks); // test_array_attributes - sm.def("ndim", [](const arr& a) { return a.ndim(); }); - sm.def("shape", [](const arr& a) { return arr(a.ndim(), a.shape()); }); - sm.def("shape", [](const arr& a, py::ssize_t dim) { return a.shape(dim); }); - sm.def("strides", [](const arr& a) { return arr(a.ndim(), a.strides()); }); - sm.def("strides", [](const arr& a, py::ssize_t dim) { return a.strides(dim); }); - sm.def("writeable", [](const arr& a) { return a.writeable(); }); - sm.def("size", [](const arr& a) { return a.size(); }); - sm.def("itemsize", [](const arr& a) { return a.itemsize(); }); - sm.def("nbytes", [](const arr& a) { return a.nbytes(); }); - sm.def("owndata", [](const arr& a) { return a.owndata(); }); + sm.def("ndim", [](const arr &a) { return a.ndim(); }); + sm.def("shape", [](const arr &a) { return arr(a.ndim(), a.shape()); }); + sm.def("shape", [](const arr &a, py::ssize_t dim) { return a.shape(dim); }); + sm.def("strides", [](const arr &a) { return arr(a.ndim(), a.strides()); }); + sm.def("strides", [](const arr &a, py::ssize_t dim) { return a.strides(dim); }); + sm.def("writeable", [](const arr &a) { return a.writeable(); }); + sm.def("size", [](const arr &a) { return a.size(); }); + sm.def("itemsize", [](const arr &a) { return a.itemsize(); }); + sm.def("nbytes", [](const arr &a) { return a.nbytes(); }); + sm.def("owndata", [](const arr &a) { return a.owndata(); }); // test_index_offset - def_index_fn(index_at, const arr&); - def_index_fn(index_at_t, const arr_t&); - def_index_fn(offset_at, const arr&); - def_index_fn(offset_at_t, const arr_t&); + def_index_fn(index_at, const arr &); + def_index_fn(index_at_t, const arr_t &); + def_index_fn(offset_at, const arr &); + def_index_fn(offset_at_t, const arr_t &); // test_data - def_index_fn(data, const arr&); - def_index_fn(data_t, const arr_t&); + def_index_fn(data, const arr &); + def_index_fn(data_t, const arr_t &); // test_mutate_data, test_mutate_readonly - def_index_fn(mutate_data, arr&); - def_index_fn(mutate_data_t, arr_t&); - def_index_fn(at_t, const arr_t&); - def_index_fn(mutate_at_t, arr_t&); + def_index_fn(mutate_data, arr &); + def_index_fn(mutate_data_t, arr_t &); + def_index_fn(at_t, const arr_t &); + def_index_fn(mutate_at_t, arr_t &); // test_make_c_f_array - sm.def("make_f_array", [] { return py::array_t<float>({ 2, 2 }, { 4, 8 }); }); - sm.def("make_c_array", [] { return py::array_t<float>({ 2, 2 }, { 8, 4 }); }); + sm.def("make_f_array", [] { return py::array_t<float>({2, 2}, {4, 8}); }); + sm.def("make_c_array", [] { return py::array_t<float>({2, 2}, {8, 4}); }); // test_empty_shaped_array sm.def("make_empty_shaped_array", [] { return py::array(py::dtype("f"), {}, {}); }); @@ -192,19 +221,17 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("scalar_int", []() { return py::array(py::dtype("i"), {}, {}, &data_i); }); // test_wrap - sm.def("wrap", [](py::array a) { - return py::array( - a.dtype(), - {a.shape(), a.shape() + a.ndim()}, - {a.strides(), a.strides() + a.ndim()}, - a.data(), - a - ); + sm.def("wrap", [](const py::array &a) { + return py::array(a.dtype(), + {a.shape(), a.shape() + a.ndim()}, + {a.strides(), a.strides() + a.ndim()}, + a.data(), + a); }); // test_numpy_view struct ArrayClass { - int data[2] = { 1, 2 }; + int data[2] = {1, 2}; ArrayClass() { py::print("ArrayClass()"); } ~ArrayClass() { py::print("~ArrayClass()"); } }; @@ -212,105 +239,121 @@ TEST_SUBMODULE(numpy_array, sm) { .def(py::init<>()) .def("numpy_view", [](py::object &obj) { py::print("ArrayClass::numpy_view()"); - auto &a = obj.cast<ArrayClass&>(); + auto &a = obj.cast<ArrayClass &>(); return py::array_t<int>({2}, {4}, a.data, obj); - } - ); + }); // test_cast_numpy_int64_to_uint64 - sm.def("function_taking_uint64", [](uint64_t) { }); + sm.def("function_taking_uint64", [](uint64_t) {}); // test_isinstance sm.def("isinstance_untyped", [](py::object yes, py::object no) { - return py::isinstance<py::array>(yes) && !py::isinstance<py::array>(no); + return py::isinstance<py::array>(std::move(yes)) + && !py::isinstance<py::array>(std::move(no)); }); - sm.def("isinstance_typed", [](py::object o) { + sm.def("isinstance_typed", [](const py::object &o) { return py::isinstance<py::array_t<double>>(o) && !py::isinstance<py::array_t<int>>(o); }); // test_constructors sm.def("default_constructors", []() { - return py::dict( - "array"_a=py::array(), - "array_t<int32>"_a=py::array_t<std::int32_t>(), - "array_t<double>"_a=py::array_t<double>() - ); + return py::dict("array"_a = py::array(), + "array_t<int32>"_a = py::array_t<std::int32_t>(), + "array_t<double>"_a = py::array_t<double>()); }); - sm.def("converting_constructors", [](py::object o) { - return py::dict( - "array"_a=py::array(o), - "array_t<int32>"_a=py::array_t<std::int32_t>(o), - "array_t<double>"_a=py::array_t<double>(o) - ); + sm.def("converting_constructors", [](const py::object &o) { + return py::dict("array"_a = py::array(o), + "array_t<int32>"_a = py::array_t<std::int32_t>(o), + "array_t<double>"_a = py::array_t<double>(o)); }); // test_overload_resolution - sm.def("overloaded", [](py::array_t<double>) { return "double"; }); - sm.def("overloaded", [](py::array_t<float>) { return "float"; }); - sm.def("overloaded", [](py::array_t<int>) { return "int"; }); - sm.def("overloaded", [](py::array_t<unsigned short>) { return "unsigned short"; }); - sm.def("overloaded", [](py::array_t<long long>) { return "long long"; }); - sm.def("overloaded", [](py::array_t<std::complex<double>>) { return "double complex"; }); - sm.def("overloaded", [](py::array_t<std::complex<float>>) { return "float complex"; }); - - sm.def("overloaded2", [](py::array_t<std::complex<double>>) { return "double complex"; }); - sm.def("overloaded2", [](py::array_t<double>) { return "double"; }); - sm.def("overloaded2", [](py::array_t<std::complex<float>>) { return "float complex"; }); - sm.def("overloaded2", [](py::array_t<float>) { return "float"; }); + sm.def("overloaded", [](const py::array_t<double> &) { return "double"; }); + sm.def("overloaded", [](const py::array_t<float> &) { return "float"; }); + sm.def("overloaded", [](const py::array_t<int> &) { return "int"; }); + sm.def("overloaded", [](const py::array_t<unsigned short> &) { return "unsigned short"; }); + sm.def("overloaded", [](const py::array_t<long long> &) { return "long long"; }); + sm.def("overloaded", + [](const py::array_t<std::complex<double>> &) { return "double complex"; }); + sm.def("overloaded", [](const py::array_t<std::complex<float>> &) { return "float complex"; }); + + sm.def("overloaded2", + [](const py::array_t<std::complex<double>> &) { return "double complex"; }); + sm.def("overloaded2", [](const py::array_t<double> &) { return "double"; }); + sm.def("overloaded2", + [](const py::array_t<std::complex<float>> &) { return "float complex"; }); + sm.def("overloaded2", [](const py::array_t<float> &) { return "float"; }); // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. // Only accept the exact types: - sm.def("overloaded3", [](py::array_t<int>) { return "int"; }, py::arg{}.noconvert()); - sm.def("overloaded3", [](py::array_t<double>) { return "double"; }, py::arg{}.noconvert()); + sm.def( + "overloaded3", [](const py::array_t<int> &) { return "int"; }, py::arg{}.noconvert()); + sm.def( + "overloaded3", + [](const py::array_t<double> &) { return "double"; }, + py::arg{}.noconvert()); // Make sure we don't do unsafe coercion (e.g. float to int) when not using forcecast, but // rather that float gets converted via the safe (conversion to double) overload: - sm.def("overloaded4", [](py::array_t<long long, 0>) { return "long long"; }); - sm.def("overloaded4", [](py::array_t<double, 0>) { return "double"; }); + sm.def("overloaded4", [](const py::array_t<long long, 0> &) { return "long long"; }); + sm.def("overloaded4", [](const py::array_t<double, 0> &) { return "double"; }); // But we do allow conversion to int if forcecast is enabled (but only if no overload matches // without conversion) - sm.def("overloaded5", [](py::array_t<unsigned int>) { return "unsigned int"; }); - sm.def("overloaded5", [](py::array_t<double>) { return "double"; }); + sm.def("overloaded5", [](const py::array_t<unsigned int> &) { return "unsigned int"; }); + sm.def("overloaded5", [](const py::array_t<double> &) { return "double"; }); // test_greedy_string_overload // Issue 685: ndarray shouldn't go to std::string overload - sm.def("issue685", [](std::string) { return "string"; }); - sm.def("issue685", [](py::array) { return "array"; }); - sm.def("issue685", [](py::object) { return "other"; }); + sm.def("issue685", [](const std::string &) { return "string"; }); + sm.def("issue685", [](const py::array &) { return "array"; }); + sm.def("issue685", [](const py::object &) { return "other"; }); // test_array_unchecked_fixed_dims - sm.def("proxy_add2", [](py::array_t<double> a, double v) { - auto r = a.mutable_unchecked<2>(); - for (py::ssize_t i = 0; i < r.shape(0); i++) - for (py::ssize_t j = 0; j < r.shape(1); j++) - r(i, j) += v; - }, py::arg{}.noconvert(), py::arg()); + sm.def( + "proxy_add2", + [](py::array_t<double> a, double v) { + auto r = a.mutable_unchecked<2>(); + for (py::ssize_t i = 0; i < r.shape(0); i++) { + for (py::ssize_t j = 0; j < r.shape(1); j++) { + r(i, j) += v; + } + } + }, + py::arg{}.noconvert(), + py::arg()); sm.def("proxy_init3", [](double start) { - py::array_t<double, py::array::c_style> a({ 3, 3, 3 }); + py::array_t<double, py::array::c_style> a({3, 3, 3}); auto r = a.mutable_unchecked<3>(); - for (py::ssize_t i = 0; i < r.shape(0); i++) - for (py::ssize_t j = 0; j < r.shape(1); j++) - for (py::ssize_t k = 0; k < r.shape(2); k++) - r(i, j, k) = start++; + for (py::ssize_t i = 0; i < r.shape(0); i++) { + for (py::ssize_t j = 0; j < r.shape(1); j++) { + for (py::ssize_t k = 0; k < r.shape(2); k++) { + r(i, j, k) = start++; + } + } + } return a; }); sm.def("proxy_init3F", [](double start) { - py::array_t<double, py::array::f_style> a({ 3, 3, 3 }); + py::array_t<double, py::array::f_style> a({3, 3, 3}); auto r = a.mutable_unchecked<3>(); - for (py::ssize_t k = 0; k < r.shape(2); k++) - for (py::ssize_t j = 0; j < r.shape(1); j++) - for (py::ssize_t i = 0; i < r.shape(0); i++) - r(i, j, k) = start++; + for (py::ssize_t k = 0; k < r.shape(2); k++) { + for (py::ssize_t j = 0; j < r.shape(1); j++) { + for (py::ssize_t i = 0; i < r.shape(0); i++) { + r(i, j, k) = start++; + } + } + } return a; }); - sm.def("proxy_squared_L2_norm", [](py::array_t<double> a) { + sm.def("proxy_squared_L2_norm", [](const py::array_t<double> &a) { auto r = a.unchecked<1>(); double sumsq = 0; - for (py::ssize_t i = 0; i < r.shape(0); i++) + for (py::ssize_t i = 0; i < r.shape(0); i++) { sumsq += r[i] * r(i); // Either notation works for a 1D array + } return sumsq; }); @@ -334,51 +377,69 @@ TEST_SUBMODULE(numpy_array, sm) { // test_array_unchecked_dyn_dims // Same as the above, but without a compile-time dimensions specification: - sm.def("proxy_add2_dyn", [](py::array_t<double> a, double v) { - auto r = a.mutable_unchecked(); - if (r.ndim() != 2) throw std::domain_error("error: ndim != 2"); - for (py::ssize_t i = 0; i < r.shape(0); i++) - for (py::ssize_t j = 0; j < r.shape(1); j++) - r(i, j) += v; - }, py::arg{}.noconvert(), py::arg()); + sm.def( + "proxy_add2_dyn", + [](py::array_t<double> a, double v) { + auto r = a.mutable_unchecked(); + if (r.ndim() != 2) { + throw std::domain_error("error: ndim != 2"); + } + for (py::ssize_t i = 0; i < r.shape(0); i++) { + for (py::ssize_t j = 0; j < r.shape(1); j++) { + r(i, j) += v; + } + } + }, + py::arg{}.noconvert(), + py::arg()); sm.def("proxy_init3_dyn", [](double start) { - py::array_t<double, py::array::c_style> a({ 3, 3, 3 }); + py::array_t<double, py::array::c_style> a({3, 3, 3}); auto r = a.mutable_unchecked(); - if (r.ndim() != 3) throw std::domain_error("error: ndim != 3"); - for (py::ssize_t i = 0; i < r.shape(0); i++) - for (py::ssize_t j = 0; j < r.shape(1); j++) - for (py::ssize_t k = 0; k < r.shape(2); k++) - r(i, j, k) = start++; + if (r.ndim() != 3) { + throw std::domain_error("error: ndim != 3"); + } + for (py::ssize_t i = 0; i < r.shape(0); i++) { + for (py::ssize_t j = 0; j < r.shape(1); j++) { + for (py::ssize_t k = 0; k < r.shape(2); k++) { + r(i, j, k) = start++; + } + } + } return a; }); sm.def("proxy_auxiliaries2_dyn", [](py::array_t<double> a) { return auxiliaries(a.unchecked(), a.mutable_unchecked()); }); - sm.def("array_auxiliaries2", [](py::array_t<double> a) { - return auxiliaries(a, a); - }); + sm.def("array_auxiliaries2", [](py::array_t<double> a) { return auxiliaries(a, a); }); // test_array_failures - // Issue #785: Uninformative "Unknown internal error" exception when constructing array from empty object: + // Issue #785: Uninformative "Unknown internal error" exception when constructing array from + // empty object: sm.def("array_fail_test", []() { return py::array(py::object()); }); sm.def("array_t_fail_test", []() { return py::array_t<double>(py::object()); }); // Make sure the error from numpy is being passed through: - sm.def("array_fail_test_negative_size", []() { int c = 0; return py::array(-1, &c); }); + sm.def("array_fail_test_negative_size", []() { + int c = 0; + return py::array(-1, &c); + }); // test_initializer_list // Issue (unnumbered; reported in #788): regression: initializer lists can be ambiguous - sm.def("array_initializer_list1", []() { return py::array_t<float>(1); }); // { 1 } also works, but clang warns about it - sm.def("array_initializer_list2", []() { return py::array_t<float>({ 1, 2 }); }); - sm.def("array_initializer_list3", []() { return py::array_t<float>({ 1, 2, 3 }); }); - sm.def("array_initializer_list4", []() { return py::array_t<float>({ 1, 2, 3, 4 }); }); + sm.def("array_initializer_list1", []() { return py::array_t<float>(1); }); + // { 1 } also works for the above, but clang warns about it + sm.def("array_initializer_list2", []() { return py::array_t<float>({1, 2}); }); + sm.def("array_initializer_list3", []() { return py::array_t<float>({1, 2, 3}); }); + sm.def("array_initializer_list4", []() { return py::array_t<float>({1, 2, 3, 4}); }); // test_array_resize // reshape array to 2D without changing size sm.def("array_reshape2", [](py::array_t<double> a) { - const auto dim_sz = (py::ssize_t)std::sqrt(a.size()); - if (dim_sz * dim_sz != a.size()) - throw std::domain_error("array_reshape2: input array total size is not a squared integer"); + const auto dim_sz = (py::ssize_t) std::sqrt(a.size()); + if (dim_sz * dim_sz != a.size()) { + throw std::domain_error( + "array_reshape2: input array total size is not a squared integer"); + } a.resize({dim_sz, dim_sz}); }); @@ -396,45 +457,96 @@ TEST_SUBMODULE(numpy_array, sm) { return a; }); - sm.def("index_using_ellipsis", [](py::array a) { - return a[py::make_tuple(0, py::ellipsis(), 0)]; + sm.def("array_view", + [](py::array_t<uint8_t> a, const std::string &dtype) { return a.view(dtype); }); + + sm.def("reshape_initializer_list", [](py::array_t<int> a, size_t N, size_t M, size_t O) { + return a.reshape({N, M, O}); }); + sm.def("reshape_tuple", [](py::array_t<int> a, const std::vector<int> &new_shape) { + return a.reshape(new_shape); + }); + + sm.def("index_using_ellipsis", + [](const py::array &a) { return a[py::make_tuple(0, py::ellipsis(), 0)]; }); // test_argument_conversions - sm.def("accept_double", - [](py::array_t<double, 0>) {}, - py::arg("a")); - sm.def("accept_double_forcecast", - [](py::array_t<double, py::array::forcecast>) {}, - py::arg("a")); - sm.def("accept_double_c_style", - [](py::array_t<double, py::array::c_style>) {}, - py::arg("a")); - sm.def("accept_double_c_style_forcecast", - [](py::array_t<double, py::array::forcecast | py::array::c_style>) {}, - py::arg("a")); - sm.def("accept_double_f_style", - [](py::array_t<double, py::array::f_style>) {}, - py::arg("a")); - sm.def("accept_double_f_style_forcecast", - [](py::array_t<double, py::array::forcecast | py::array::f_style>) {}, - py::arg("a")); - sm.def("accept_double_noconvert", - [](py::array_t<double, 0>) {}, - "a"_a.noconvert()); - sm.def("accept_double_forcecast_noconvert", - [](py::array_t<double, py::array::forcecast>) {}, - "a"_a.noconvert()); - sm.def("accept_double_c_style_noconvert", - [](py::array_t<double, py::array::c_style>) {}, - "a"_a.noconvert()); - sm.def("accept_double_c_style_forcecast_noconvert", - [](py::array_t<double, py::array::forcecast | py::array::c_style>) {}, - "a"_a.noconvert()); - sm.def("accept_double_f_style_noconvert", - [](py::array_t<double, py::array::f_style>) {}, - "a"_a.noconvert()); - sm.def("accept_double_f_style_forcecast_noconvert", - [](py::array_t<double, py::array::forcecast | py::array::f_style>) {}, - "a"_a.noconvert()); + sm.def( + "accept_double", [](const py::array_t<double, 0> &) {}, py::arg("a")); + sm.def( + "accept_double_forcecast", + [](const py::array_t<double, py::array::forcecast> &) {}, + py::arg("a")); + sm.def( + "accept_double_c_style", + [](const py::array_t<double, py::array::c_style> &) {}, + py::arg("a")); + sm.def( + "accept_double_c_style_forcecast", + [](const py::array_t<double, py::array::forcecast | py::array::c_style> &) {}, + py::arg("a")); + sm.def( + "accept_double_f_style", + [](const py::array_t<double, py::array::f_style> &) {}, + py::arg("a")); + sm.def( + "accept_double_f_style_forcecast", + [](const py::array_t<double, py::array::forcecast | py::array::f_style> &) {}, + py::arg("a")); + sm.def( + "accept_double_noconvert", [](const py::array_t<double, 0> &) {}, "a"_a.noconvert()); + sm.def( + "accept_double_forcecast_noconvert", + [](const py::array_t<double, py::array::forcecast> &) {}, + "a"_a.noconvert()); + sm.def( + "accept_double_c_style_noconvert", + [](const py::array_t<double, py::array::c_style> &) {}, + "a"_a.noconvert()); + sm.def( + "accept_double_c_style_forcecast_noconvert", + [](const py::array_t<double, py::array::forcecast | py::array::c_style> &) {}, + "a"_a.noconvert()); + sm.def( + "accept_double_f_style_noconvert", + [](const py::array_t<double, py::array::f_style> &) {}, + "a"_a.noconvert()); + sm.def( + "accept_double_f_style_forcecast_noconvert", + [](const py::array_t<double, py::array::forcecast | py::array::f_style> &) {}, + "a"_a.noconvert()); + + // Check that types returns correct npy format descriptor + sm.def("test_fmt_desc_float", [](const py::array_t<float> &) {}); + sm.def("test_fmt_desc_double", [](const py::array_t<double> &) {}); + sm.def("test_fmt_desc_const_float", [](const py::array_t<const float> &) {}); + sm.def("test_fmt_desc_const_double", [](const py::array_t<const double> &) {}); + + sm.def("round_trip_float", [](double d) { return d; }); + + sm.def("pass_array_pyobject_ptr_return_sum_str_values", + [](const py::array_t<PyObject *> &objs) { + std::string sum_str_values; + for (const auto &obj : objs) { + sum_str_values += py::str(obj.attr("value")); + } + return sum_str_values; + }); + + sm.def("pass_array_pyobject_ptr_return_as_list", + [](const py::array_t<PyObject *> &objs) -> py::list { return objs; }); + + sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) { + py::size_t arr_size = py::len(objs); + py::array_t<PyObject *> arr_from_list(static_cast<py::ssize_t>(arr_size)); + PyObject **data = arr_from_list.mutable_data(); + for (py::size_t i = 0; i < arr_size; i++) { + assert(data[i] == nullptr); + data[i] = py::cast<PyObject *>(objs[i].attr("value")); + } + return arr_from_list; + }); + + sm.def("return_array_pyobject_ptr_from_list", + [](const py::list &objs) -> py::array_t<PyObject *> { return objs; }); } diff --git a/tests/test_numpy_array.py b/tests/test_numpy_array.py index 02f3ecfc..12e7d17d 100644 --- a/tests/test_numpy_array.py +++ b/tests/test_numpy_array.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- import pytest import env # noqa: F401 - from pybind11_tests import numpy_array as m np = pytest.importorskip("numpy") @@ -20,13 +18,11 @@ def test_dtypes(): assert check.numpy == check.pybind11, check if check.numpy.num != check.pybind11.num: print( - "NOTE: typenum mismatch for {}: {} != {}".format( - check, check.numpy.num, check.pybind11.num - ) + f"NOTE: typenum mismatch for {check}: {check.numpy.num} != {check.pybind11.num}" ) -@pytest.fixture(scope="function") +@pytest.fixture() def arr(): return np.array([[1, 2, 3], [4, 5, 6]], "=u2") @@ -71,7 +67,7 @@ def test_array_attributes(): @pytest.mark.parametrize( - "args, ret", [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] + ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] ) def test_index_offset(arr, args, ret): assert m.index_at(arr, *args) == ret @@ -97,7 +93,7 @@ def test_dim_check_fail(arr): @pytest.mark.parametrize( - "args, ret", + ("args", "ret"), [ ([], [1, 2, 3, 4, 5, 6]), ([1], [4, 5, 6]), @@ -118,9 +114,7 @@ def test_at_fail(arr, dim): for func in m.at_t, m.mutate_at_t: with pytest.raises(IndexError) as excinfo: func(arr, *([0] * dim)) - assert str(excinfo.value) == "index dimension mismatch: {} (ndim = 2)".format( - dim - ) + assert str(excinfo.value) == f"index dimension mismatch: {dim} (ndim = 2)" def test_at(arr): @@ -194,8 +188,6 @@ def test_make_empty_shaped_array(): def test_wrap(): def assert_references(a, b, base=None): - from distutils.version import LooseVersion - if base is None: base = a assert a is not b @@ -206,7 +198,8 @@ def test_wrap(): assert a.flags.f_contiguous == b.flags.f_contiguous assert a.flags.writeable == b.flags.writeable assert a.flags.aligned == b.flags.aligned - if LooseVersion(np.__version__) >= LooseVersion("1.14.0"): + # 1.13 supported Python 3.6 + if tuple(int(x) for x in np.__version__.split(".")[:2]) >= (1, 14): assert a.flags.writebackifcopy == b.flags.writebackifcopy else: assert a.flags.updateifcopy == b.flags.updateifcopy @@ -218,12 +211,14 @@ def test_wrap(): assert b[0, 0] == 1234 a1 = np.array([1, 2], dtype=np.int16) - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="F") - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) @@ -412,7 +407,7 @@ def test_array_unchecked_fixed_dims(msg): assert m.proxy_auxiliaries2_const_ref(z1) -def test_array_unchecked_dyn_dims(msg): +def test_array_unchecked_dyn_dims(): z1 = np.array([[1, 2], [3, 4]], dtype="float64") m.proxy_add2_dyn(z1, 10) assert np.all(z1 == [[11, 12], [13, 14]]) @@ -445,7 +440,7 @@ def test_initializer_list(): assert m.array_initializer_list4().shape == (1, 2, 3, 4) -def test_array_resize(msg): +def test_array_resize(): a = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype="float64") m.array_reshape2(a) assert a.size == 9 @@ -458,30 +453,85 @@ def test_array_resize(msg): try: m.array_resize3(a, 3, True) except ValueError as e: - assert str(e).startswith("cannot resize an array") + assert str(e).startswith("cannot resize an array") # noqa: PT017 # transposed array doesn't own data b = a.transpose() try: m.array_resize3(b, 3, False) except ValueError as e: - assert str(e).startswith("cannot resize this array: it does not own its data") + assert str(e).startswith( # noqa: PT017 + "cannot resize this array: it does not own its data" + ) # ... but reshape should be fine m.array_reshape2(b) assert b.shape == (8, 8) @pytest.mark.xfail("env.PYPY") -def test_array_create_and_resize(msg): +def test_array_create_and_resize(): a = m.create_and_resize(2) assert a.size == 4 assert np.all(a == 42.0) +def test_array_view(): + a = np.ones(100 * 4).astype("uint8") + a_float_view = m.array_view(a, "float32") + assert a_float_view.shape == (100 * 1,) # 1 / 4 bytes = 8 / 32 + + a_int16_view = m.array_view(a, "int16") # 1 / 2 bytes = 16 / 32 + assert a_int16_view.shape == (100 * 2,) + + +def test_array_view_invalid(): + a = np.ones(100 * 4).astype("uint8") + with pytest.raises(TypeError): + m.array_view(a, "deadly_dtype") + + +def test_reshape_initializer_list(): + a = np.arange(2 * 7 * 3) + 1 + x = m.reshape_initializer_list(a, 2, 7, 3) + assert x.shape == (2, 7, 3) + assert list(x[1][4]) == [34, 35, 36] + with pytest.raises(ValueError) as excinfo: + m.reshape_initializer_list(a, 1, 7, 3) + assert str(excinfo.value) == "cannot reshape array of size 42 into shape (1,7,3)" + + +def test_reshape_tuple(): + a = np.arange(3 * 7 * 2) + 1 + x = m.reshape_tuple(a, (3, 7, 2)) + assert x.shape == (3, 7, 2) + assert list(x[1][4]) == [23, 24] + y = m.reshape_tuple(x, (x.size,)) + assert y.shape == (42,) + with pytest.raises(ValueError) as excinfo: + m.reshape_tuple(a, (3, 7, 1)) + assert str(excinfo.value) == "cannot reshape array of size 42 into shape (3,7,1)" + with pytest.raises(ValueError) as excinfo: + m.reshape_tuple(a, ()) + assert str(excinfo.value) == "cannot reshape array of size 42 into shape ()" + + def test_index_using_ellipsis(): a = m.index_using_ellipsis(np.zeros((5, 6, 7))) assert a.shape == (6,) +@pytest.mark.parametrize( + "test_func", + [ + m.test_fmt_desc_float, + m.test_fmt_desc_double, + m.test_fmt_desc_const_float, + m.test_fmt_desc_const_double, + ], +) +def test_format_descriptors_for_floating_point_types(test_func): + assert "numpy.ndarray[numpy.float" in test_func.__doc__ + + @pytest.mark.parametrize("forcecast", [False, True]) @pytest.mark.parametrize("contiguity", [None, "C", "F"]) @pytest.mark.parametrize("noconvert", [False, True]) @@ -539,3 +589,80 @@ def test_dtype_refcount_leak(): m.ndim(a) after = getrefcount(dtype) assert after == before + + +def test_round_trip_float(): + arr = np.zeros((), np.float64) + arr[()] = 37.2 + assert m.round_trip_float(arr) == 37.2 + + +# HINT: An easy and robust way (although only manual unfortunately) to check for +# ref-count leaks in the test_.*pyobject_ptr.* functions below is to +# * temporarily insert `while True:` (one-by-one), +# * run this test, and +# * run the Linux `top` command in another shell to visually monitor +# `RES` for a minute or two. +# If there is a leak, it is usually evident in seconds because the `RES` +# value increases without bounds. (Don't forget to Ctrl-C the test!) + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below: +# * Ref-count leaks will be immediately evident. +# * Sanitizers are much more likely to detect heap-use-after-free due to +# other ref-count bugs. +class PyValueHolder: + def __init__(self, value): + self.value = value + + +def WrapWithPyValueHolder(*values): + return [PyValueHolder(v) for v in values] + + +def UnwrapPyValueHolder(vhs): + return [vh.value for vh in vhs] + + +def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object) + ) + == "-3four5.0" + ) + + +def test_pass_array_pyobject_ptr_return_sum_str_values_list(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + WrapWithPyValueHolder(2, "three", -4.0) + ) + == "2three-4.0" + ) + + +def test_pass_array_pyobject_ptr_return_as_list(): + # Intentionally all temporaries, do not change. + assert UnwrapPyValueHolder( + m.pass_array_pyobject_ptr_return_as_list( + np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object) + ) + ) == [-1, "two", 3.0] + + +@pytest.mark.parametrize( + ("return_array_pyobject_ptr", "unwrap"), + [ + (m.return_array_pyobject_ptr_cpp_loop, list), + (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder), + ], +) +def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap): + # Intentionally all temporaries, do not change. + arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0)) + assert isinstance(arr_from_list, np.ndarray) + assert arr_from_list.dtype == np.dtype("O") + assert unwrap(arr_from_list) == [6, "seven", -8.0] diff --git a/tests/test_numpy_dtypes.cpp b/tests/test_numpy_dtypes.cpp index b2e5e607..6654f9ed 100644 --- a/tests/test_numpy_dtypes.cpp +++ b/tests/test_numpy_dtypes.cpp @@ -7,13 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/numpy.h> +#include "pybind11_tests.h" + #ifdef __GNUC__ -#define PYBIND11_PACKED(cls) cls __attribute__((__packed__)) +# define PYBIND11_PACKED(cls) cls __attribute__((__packed__)) #else -#define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop)) +# define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop)) #endif namespace py = pybind11; @@ -25,7 +26,7 @@ struct SimpleStruct { long double ldbl_; }; -std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) { +std::ostream &operator<<(std::ostream &os, const SimpleStruct &v) { return os << "s:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } @@ -43,7 +44,7 @@ PYBIND11_PACKED(struct PackedStruct { long double ldbl_; }); -std::ostream& operator<<(std::ostream& os, const PackedStruct& v) { +std::ostream &operator<<(std::ostream &os, const PackedStruct &v) { return os << "p:" << v.bool_ << "," << v.uint_ << "," << v.float_ << "," << v.ldbl_; } @@ -52,7 +53,7 @@ PYBIND11_PACKED(struct NestedStruct { PackedStruct b; }); -std::ostream& operator<<(std::ostream& os, const NestedStruct& v) { +std::ostream &operator<<(std::ostream &os, const NestedStruct &v) { return os << "n:a=" << v.a << ";b=" << v.b; } @@ -70,7 +71,7 @@ struct PartialNestedStruct { uint64_t dummy2; }; -struct UnboundStruct { }; +struct UnboundStruct {}; struct StringStruct { char a[3]; @@ -82,7 +83,7 @@ struct ComplexStruct { std::complex<double> cdbl; }; -std::ostream& operator<<(std::ostream& os, const ComplexStruct& v) { +std::ostream &operator<<(std::ostream &os, const ComplexStruct &v) { return os << "c:" << v.cflt << "," << v.cdbl; } @@ -106,57 +107,65 @@ PYBIND11_PACKED(struct EnumStruct { E2 e2; }); -std::ostream& operator<<(std::ostream& os, const StringStruct& v) { +std::ostream &operator<<(std::ostream &os, const StringStruct &v) { os << "a='"; - for (size_t i = 0; i < 3 && v.a[i]; i++) os << v.a[i]; + for (size_t i = 0; i < 3 && (v.a[i] != 0); i++) { + os << v.a[i]; + } os << "',b='"; - for (size_t i = 0; i < 3 && v.b[i]; i++) os << v.b[i]; + for (size_t i = 0; i < 3 && (v.b[i] != 0); i++) { + os << v.b[i]; + } return os << "'"; } -std::ostream& operator<<(std::ostream& os, const ArrayStruct& v) { +std::ostream &operator<<(std::ostream &os, const ArrayStruct &v) { os << "a={"; for (int i = 0; i < 3; i++) { - if (i > 0) + if (i > 0) { os << ','; + } os << '{'; - for (int j = 0; j < 3; j++) + for (int j = 0; j < 3; j++) { os << v.a[i][j] << ','; + } os << v.a[i][3] << '}'; } os << "},b={" << v.b[0] << ',' << v.b[1]; os << "},c={" << int(v.c[0]) << ',' << int(v.c[1]) << ',' << int(v.c[2]); os << "},d={"; for (int i = 0; i < 4; i++) { - if (i > 0) + if (i > 0) { os << ','; + } os << '{' << v.d[i][0] << ',' << v.d[i][1] << '}'; } return os << '}'; } -std::ostream& operator<<(std::ostream& os, const EnumStruct& v) { +std::ostream &operator<<(std::ostream &os, const EnumStruct &v) { return os << "e1=" << (v.e1 == E1::A ? "A" : "B") << ",e2=" << (v.e2 == E2::X ? "X" : "Y"); } template <typename T> py::array mkarray_via_buffer(size_t n) { - return py::array(py::buffer_info(nullptr, sizeof(T), - py::format_descriptor<T>::format(), - 1, { n }, { sizeof(T) })); + return py::array(py::buffer_info( + nullptr, sizeof(T), py::format_descriptor<T>::format(), 1, {n}, {sizeof(T)})); } -#define SET_TEST_VALS(s, i) do { \ - s.bool_ = (i) % 2 != 0; \ - s.uint_ = (uint32_t) (i); \ - s.float_ = (float) (i) * 1.5f; \ - s.ldbl_ = (long double) (i) * -2.5L; } while (0) +#define SET_TEST_VALS(s, i) \ + do { \ + (s).bool_ = (i) % 2 != 0; \ + (s).uint_ = (uint32_t) (i); \ + (s).float_ = (float) (i) *1.5f; \ + (s).ldbl_ = (long double) (i) * -2.5L; \ + } while (0) template <typename S> py::array_t<S, 0> create_recarray(size_t n) { auto arr = mkarray_via_buffer<S>(n); auto req = arr.request(); - auto ptr = static_cast<S*>(req.ptr); + auto *ptr = static_cast<S *>(req.ptr); for (size_t i = 0; i < n; i++) { SET_TEST_VALS(ptr[i], i); } @@ -166,7 +175,7 @@ py::array_t<S, 0> create_recarray(size_t n) { template <typename S> py::list print_recarray(py::array_t<S, 0> arr) { const auto req = arr.request(); - const auto ptr = static_cast<S*>(req.ptr); + auto *const ptr = static_cast<S *>(req.ptr); auto l = py::list(); for (py::ssize_t i = 0; i < req.size; i++) { std::stringstream ss; @@ -179,12 +188,12 @@ py::list print_recarray(py::array_t<S, 0> arr) { py::array_t<int32_t, 0> test_array_ctors(int i) { using arr_t = py::array_t<int32_t, 0>; - std::vector<int32_t> data { 1, 2, 3, 4, 5, 6 }; - std::vector<py::ssize_t> shape { 3, 2 }; - std::vector<py::ssize_t> strides { 8, 4 }; + std::vector<int32_t> data{1, 2, 3, 4, 5, 6}; + std::vector<py::ssize_t> shape{3, 2}; + std::vector<py::ssize_t> strides{8, 4}; - auto ptr = data.data(); - auto vptr = (void *) ptr; + auto *ptr = data.data(); + auto *vptr = (void *) ptr; auto dtype = py::dtype("int32"); py::buffer_info buf_ndim1(vptr, 4, "i", 6); @@ -194,41 +203,69 @@ py::array_t<int32_t, 0> test_array_ctors(int i) { auto fill = [](py::array arr) { auto req = arr.request(); - for (int i = 0; i < 6; i++) ((int32_t *) req.ptr)[i] = i + 1; + for (int i = 0; i < 6; i++) { + ((int32_t *) req.ptr)[i] = i + 1; + } return arr; }; switch (i) { - // shape: (3, 2) - case 10: return arr_t(shape, strides, ptr); - case 11: return py::array(shape, strides, ptr); - case 12: return py::array(dtype, shape, strides, vptr); - case 13: return arr_t(shape, ptr); - case 14: return py::array(shape, ptr); - case 15: return py::array(dtype, shape, vptr); - case 16: return arr_t(buf_ndim2); - case 17: return py::array(buf_ndim2); - // shape: (3, 2) - post-fill - case 20: return fill(arr_t(shape, strides)); - case 21: return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor - case 22: return fill(py::array(dtype, shape, strides)); - case 23: return fill(arr_t(shape)); - case 24: return py::array(shape, ptr); // can't have nullptr due to templated ctor - case 25: return fill(py::array(dtype, shape)); - case 26: return fill(arr_t(buf_ndim2_null)); - case 27: return fill(py::array(buf_ndim2_null)); - // shape: (6, ) - case 30: return arr_t(6, ptr); - case 31: return py::array(6, ptr); - case 32: return py::array(dtype, 6, vptr); - case 33: return arr_t(buf_ndim1); - case 34: return py::array(buf_ndim1); - // shape: (6, ) - case 40: return fill(arr_t(6)); - case 41: return py::array(6, ptr); // can't have nullptr due to templated ctor - case 42: return fill(py::array(dtype, 6)); - case 43: return fill(arr_t(buf_ndim1_null)); - case 44: return fill(py::array(buf_ndim1_null)); + // shape: (3, 2) + case 10: + return arr_t(shape, strides, ptr); + case 11: + return py::array(shape, strides, ptr); + case 12: + return py::array(dtype, shape, strides, vptr); + case 13: + return arr_t(shape, ptr); + case 14: + return py::array(shape, ptr); + case 15: + return py::array(dtype, shape, vptr); + case 16: + return arr_t(buf_ndim2); + case 17: + return py::array(buf_ndim2); + // shape: (3, 2) - post-fill + case 20: + return fill(arr_t(shape, strides)); + case 21: + return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor + case 22: + return fill(py::array(dtype, shape, strides)); + case 23: + return fill(arr_t(shape)); + case 24: + return py::array(shape, ptr); // can't have nullptr due to templated ctor + case 25: + return fill(py::array(dtype, shape)); + case 26: + return fill(arr_t(buf_ndim2_null)); + case 27: + return fill(py::array(buf_ndim2_null)); + // shape: (6, ) + case 30: + return arr_t(6, ptr); + case 31: + return py::array(6, ptr); + case 32: + return py::array(dtype, 6, vptr); + case 33: + return arr_t(buf_ndim1); + case 34: + return py::array(buf_ndim1); + // shape: (6, ) + case 40: + return fill(arr_t(6)); + case 41: + return py::array(6, ptr); // can't have nullptr due to templated ctor + case 42: + return fill(py::array(dtype, 6)); + case 43: + return fill(arr_t(buf_ndim1_null)); + case 44: + return fill(py::array(buf_ndim1_null)); } return arr_t(); } @@ -240,14 +277,21 @@ py::list test_dtype_ctors() { list.append(py::dtype::from_args(py::str("bool"))); py::list names, offsets, formats; py::dict dict; - names.append(py::str("a")); names.append(py::str("b")); dict["names"] = names; - offsets.append(py::int_(1)); offsets.append(py::int_(10)); dict["offsets"] = offsets; - formats.append(py::dtype("int32")); formats.append(py::dtype("float64")); dict["formats"] = formats; + names.append(py::str("a")); + names.append(py::str("b")); + dict["names"] = names; + offsets.append(py::int_(1)); + offsets.append(py::int_(10)); + dict["offsets"] = offsets; + formats.append(py::dtype("int32")); + formats.append(py::dtype("float64")); + dict["formats"] = formats; dict["itemsize"] = py::int_(20); list.append(py::dtype::from_args(dict)); list.append(py::dtype(names, formats, offsets, 20)); - list.append(py::dtype(py::buffer_info((void *) 0, sizeof(unsigned int), "I", 1))); - list.append(py::dtype(py::buffer_info((void *) 0, 0, "T{i:a:f:b:}", 1))); + list.append(py::dtype(py::buffer_info((void *) nullptr, sizeof(unsigned int), "I", 1))); + list.append(py::dtype(py::buffer_info((void *) nullptr, 0, "T{i:a:f:b:}", 1))); + list.append(py::dtype(py::detail::npy_api::NPY_DOUBLE_)); return list; } @@ -255,8 +299,11 @@ struct A {}; struct B {}; TEST_SUBMODULE(numpy_dtypes, m) { - try { py::module_::import("numpy"); } - catch (...) { return; } + try { + py::module_::import("numpy"); + } catch (const py::error_already_set &) { + return; + } // typeinfo may be registered before the dtype descriptor for scalar casts to work... py::class_<SimpleStruct>(m, "SimpleStruct") @@ -266,18 +313,18 @@ TEST_SUBMODULE(numpy_dtypes, m) { .def_readwrite("uint_", &SimpleStruct::uint_) .def_readwrite("float_", &SimpleStruct::float_) .def_readwrite("ldbl_", &SimpleStruct::ldbl_) - .def("astuple", [](const SimpleStruct& self) { - return py::make_tuple(self.bool_, self.uint_, self.float_, self.ldbl_); - }) - .def_static("fromtuple", [](const py::tuple tup) { + .def("astuple", + [](const SimpleStruct &self) { + return py::make_tuple(self.bool_, self.uint_, self.float_, self.ldbl_); + }) + .def_static("fromtuple", [](const py::tuple &tup) { if (py::len(tup) != 4) { throw py::cast_error("Invalid size"); } - return SimpleStruct{ - tup[0].cast<bool>(), - tup[1].cast<uint32_t>(), - tup[2].cast<float>(), - tup[3].cast<long double>()}; + return SimpleStruct{tup[0].cast<bool>(), + tup[1].cast<uint32_t>(), + tup[2].cast<float>(), + tup[3].cast<long double>()}; }); PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); @@ -296,19 +343,21 @@ TEST_SUBMODULE(numpy_dtypes, m) { PYBIND11_NUMPY_DTYPE_EX(StructWithUglyNames, __x__, "x", __y__, "y"); - // If uncommented, this should produce a static_assert failure telling the user that the struct +#ifdef PYBIND11_NEVER_DEFINED_EVER + // If enabled, this should produce a static_assert failure telling the user that the struct // is not a POD type -// struct NotPOD { std::string v; NotPOD() : v("hi") {}; }; -// PYBIND11_NUMPY_DTYPE(NotPOD, v); + struct NotPOD { + std::string v; + NotPOD() : v("hi"){}; + }; + PYBIND11_NUMPY_DTYPE(NotPOD, v); +#endif // Check that dtypes can be registered programmatically, both from // initializer lists of field descriptors and from other containers. - py::detail::npy_format_descriptor<A>::register_dtype( - {} - ); + py::detail::npy_format_descriptor<A>::register_dtype({}); py::detail::npy_format_descriptor<B>::register_dtype( - std::vector<py::detail::field_descriptor>{} - ); + std::vector<py::detail::field_descriptor>{}); // test_recarray, test_scalar_conversion m.def("create_rec_simple", &create_recarray<SimpleStruct>); @@ -316,7 +365,7 @@ TEST_SUBMODULE(numpy_dtypes, m) { m.def("create_rec_nested", [](size_t n) { // test_signature py::array_t<NestedStruct, 0> arr = mkarray_via_buffer<NestedStruct>(n); auto req = arr.request(); - auto ptr = static_cast<NestedStruct*>(req.ptr); + auto *ptr = static_cast<NestedStruct *>(req.ptr); for (size_t i = 0; i < n; i++) { SET_TEST_VALS(ptr[i].a, i); SET_TEST_VALS(ptr[i].b, i + 1); @@ -327,7 +376,7 @@ TEST_SUBMODULE(numpy_dtypes, m) { m.def("create_rec_partial_nested", [](size_t n) { py::array_t<PartialNestedStruct, 0> arr = mkarray_via_buffer<PartialNestedStruct>(n); auto req = arr.request(); - auto ptr = static_cast<PartialNestedStruct*>(req.ptr); + auto *ptr = static_cast<PartialNestedStruct *>(req.ptr); for (size_t i = 0; i < n; i++) { SET_TEST_VALS(ptr[i].a, i); } @@ -341,48 +390,95 @@ TEST_SUBMODULE(numpy_dtypes, m) { m.def("get_format_unbound", []() { return py::format_descriptor<UnboundStruct>::format(); }); m.def("print_format_descriptors", []() { py::list l; - for (const auto &fmt : { - py::format_descriptor<SimpleStruct>::format(), - py::format_descriptor<PackedStruct>::format(), - py::format_descriptor<NestedStruct>::format(), - py::format_descriptor<PartialStruct>::format(), - py::format_descriptor<PartialNestedStruct>::format(), - py::format_descriptor<StringStruct>::format(), - py::format_descriptor<ArrayStruct>::format(), - py::format_descriptor<EnumStruct>::format(), - py::format_descriptor<ComplexStruct>::format() - }) { + for (const auto &fmt : {py::format_descriptor<SimpleStruct>::format(), + py::format_descriptor<PackedStruct>::format(), + py::format_descriptor<NestedStruct>::format(), + py::format_descriptor<PartialStruct>::format(), + py::format_descriptor<PartialNestedStruct>::format(), + py::format_descriptor<StringStruct>::format(), + py::format_descriptor<ArrayStruct>::format(), + py::format_descriptor<EnumStruct>::format(), + py::format_descriptor<ComplexStruct>::format()}) { l.append(py::cast(fmt)); } return l; }); // test_dtype + std::vector<const char *> dtype_names{ + "byte", "short", "intc", "int_", "longlong", "ubyte", "ushort", + "uintc", "uint", "ulonglong", "half", "single", "double", "longdouble", + "csingle", "cdouble", "clongdouble", "bool_", "datetime64", "timedelta64", "object_"}; + m.def("print_dtypes", []() { py::list l; - for (const py::handle &d : { - py::dtype::of<SimpleStruct>(), - py::dtype::of<PackedStruct>(), - py::dtype::of<NestedStruct>(), - py::dtype::of<PartialStruct>(), - py::dtype::of<PartialNestedStruct>(), - py::dtype::of<StringStruct>(), - py::dtype::of<ArrayStruct>(), - py::dtype::of<EnumStruct>(), - py::dtype::of<StructWithUglyNames>(), - py::dtype::of<ComplexStruct>() - }) + for (const py::handle &d : {py::dtype::of<SimpleStruct>(), + py::dtype::of<PackedStruct>(), + py::dtype::of<NestedStruct>(), + py::dtype::of<PartialStruct>(), + py::dtype::of<PartialNestedStruct>(), + py::dtype::of<StringStruct>(), + py::dtype::of<ArrayStruct>(), + py::dtype::of<EnumStruct>(), + py::dtype::of<StructWithUglyNames>(), + py::dtype::of<ComplexStruct>()}) { l.append(py::str(d)); + } return l; }); m.def("test_dtype_ctors", &test_dtype_ctors); + m.def("test_dtype_kind", [dtype_names]() { + py::list list; + for (const auto &dt_name : dtype_names) { + list.append(py::dtype(dt_name).kind()); + } + return list; + }); + m.def("test_dtype_char_", [dtype_names]() { + py::list list; + for (const auto &dt_name : dtype_names) { + list.append(py::dtype(dt_name).char_()); + } + return list; + }); + m.def("test_dtype_num", [dtype_names]() { + py::list list; + for (const auto &dt_name : dtype_names) { + list.append(py::dtype(dt_name).num()); + } + return list; + }); + m.def("test_dtype_byteorder", [dtype_names]() { + py::list list; + for (const auto &dt_name : dtype_names) { + list.append(py::dtype(dt_name).byteorder()); + } + return list; + }); + m.def("test_dtype_alignment", [dtype_names]() { + py::list list; + for (const auto &dt_name : dtype_names) { + list.append(py::dtype(dt_name).alignment()); + } + return list; + }); + m.def("test_dtype_flags", [dtype_names]() { + py::list list; + for (const auto &dt_name : dtype_names) { + list.append(py::dtype(dt_name).flags()); + } + return list; + }); m.def("test_dtype_methods", []() { py::list list; auto dt1 = py::dtype::of<int32_t>(); auto dt2 = py::dtype::of<SimpleStruct>(); - list.append(dt1); list.append(dt2); - list.append(py::bool_(dt1.has_fields())); list.append(py::bool_(dt2.has_fields())); - list.append(py::int_(dt1.itemsize())); list.append(py::int_(dt2.itemsize())); + list.append(dt1); + list.append(dt2); + list.append(py::bool_(dt1.has_fields())); + list.append(py::bool_(dt2.has_fields())); + list.append(py::int_(dt1.itemsize())); + list.append(py::int_(dt2.itemsize())); return list; }); struct TrailingPaddingStruct { @@ -397,17 +493,24 @@ TEST_SUBMODULE(numpy_dtypes, m) { py::array_t<StringStruct, 0> arr = mkarray_via_buffer<StringStruct>(non_empty ? 4 : 0); if (non_empty) { auto req = arr.request(); - auto ptr = static_cast<StringStruct*>(req.ptr); - for (py::ssize_t i = 0; i < req.size * req.itemsize; i++) - static_cast<char*>(req.ptr)[i] = 0; - ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a'; - ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a'; - ptr[3].a[0] = 'a'; ptr[3].b[0] = 'a'; - - ptr[2].a[1] = 'b'; ptr[2].b[1] = 'b'; - ptr[3].a[1] = 'b'; ptr[3].b[1] = 'b'; - - ptr[3].a[2] = 'c'; ptr[3].b[2] = 'c'; + auto *ptr = static_cast<StringStruct *>(req.ptr); + for (py::ssize_t i = 0; i < req.size * req.itemsize; i++) { + static_cast<char *>(req.ptr)[i] = 0; + } + ptr[1].a[0] = 'a'; + ptr[1].b[0] = 'a'; + ptr[2].a[0] = 'a'; + ptr[2].b[0] = 'a'; + ptr[3].a[0] = 'a'; + ptr[3].b[0] = 'a'; + + ptr[2].a[1] = 'b'; + ptr[2].b[1] = 'b'; + ptr[3].a[1] = 'b'; + ptr[3].b[1] = 'b'; + + ptr[3].a[2] = 'c'; + ptr[3].b[2] = 'c'; } return arr; }); @@ -416,18 +519,24 @@ TEST_SUBMODULE(numpy_dtypes, m) { // test_array_array m.def("create_array_array", [](size_t n) { py::array_t<ArrayStruct, 0> arr = mkarray_via_buffer<ArrayStruct>(n); - auto ptr = (ArrayStruct *) arr.mutable_data(); + auto *ptr = (ArrayStruct *) arr.mutable_data(); for (size_t i = 0; i < n; i++) { - for (size_t j = 0; j < 3; j++) - for (size_t k = 0; k < 4; k++) + for (size_t j = 0; j < 3; j++) { + for (size_t k = 0; k < 4; k++) { ptr[i].a[j][k] = char('A' + (i * 100 + j * 10 + k) % 26); - for (size_t j = 0; j < 2; j++) + } + } + for (size_t j = 0; j < 2; j++) { ptr[i].b[j] = int32_t(i * 1000 + j); - for (size_t j = 0; j < 3; j++) + } + for (size_t j = 0; j < 3; j++) { ptr[i].c[j] = uint8_t(i * 10 + j); - for (size_t j = 0; j < 4; j++) - for (size_t k = 0; k < 2; k++) + } + for (size_t j = 0; j < 4; j++) { + for (size_t k = 0; k < 2; k++) { ptr[i].d[j][k] = float(i) * 100.0f + float(j) * 10.0f + float(k); + } + } } return arr; }); @@ -436,7 +545,7 @@ TEST_SUBMODULE(numpy_dtypes, m) { // test_enum_array m.def("create_enum_array", [](size_t n) { py::array_t<EnumStruct, 0> arr = mkarray_via_buffer<EnumStruct>(n); - auto ptr = (EnumStruct *) arr.mutable_data(); + auto *ptr = (EnumStruct *) arr.mutable_data(); for (size_t i = 0; i < n; i++) { ptr[i].e1 = static_cast<E1>(-1 + ((int) i % 2) * 2); ptr[i].e2 = static_cast<E2>(1 + (i % 2)); @@ -448,7 +557,7 @@ TEST_SUBMODULE(numpy_dtypes, m) { // test_complex_array m.def("create_complex_array", [](size_t n) { py::array_t<ComplexStruct, 0> arr = mkarray_via_buffer<ComplexStruct>(n); - auto ptr = (ComplexStruct *) arr.mutable_data(); + auto *ptr = (ComplexStruct *) arr.mutable_data(); for (size_t i = 0; i < n; i++) { ptr[i].cflt.real(float(i)); ptr[i].cflt.imag(float(i) + 0.25f); @@ -471,14 +580,19 @@ TEST_SUBMODULE(numpy_dtypes, m) { PYBIND11_NUMPY_DTYPE(CompareStruct, x, y, z); m.def("compare_buffer_info", []() { py::list list; - list.append(py::bool_(py::detail::compare_buffer_info<float>::compare(py::buffer_info(nullptr, sizeof(float), "f", 1)))); - list.append(py::bool_(py::detail::compare_buffer_info<unsigned>::compare(py::buffer_info(nullptr, sizeof(int), "I", 1)))); - list.append(py::bool_(py::detail::compare_buffer_info<long>::compare(py::buffer_info(nullptr, sizeof(long), "l", 1)))); - list.append(py::bool_(py::detail::compare_buffer_info<long>::compare(py::buffer_info(nullptr, sizeof(long), sizeof(long) == sizeof(int) ? "i" : "q", 1)))); - list.append(py::bool_(py::detail::compare_buffer_info<CompareStruct>::compare(py::buffer_info(nullptr, sizeof(CompareStruct), "T{?:x:3xI:y:f:z:}", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<float>::compare( + py::buffer_info(nullptr, sizeof(float), "f", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<unsigned>::compare( + py::buffer_info(nullptr, sizeof(int), "I", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<long>::compare( + py::buffer_info(nullptr, sizeof(long), "l", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<long>::compare( + py::buffer_info(nullptr, sizeof(long), sizeof(long) == sizeof(int) ? "i" : "q", 1)))); + list.append(py::bool_(py::detail::compare_buffer_info<CompareStruct>::compare( + py::buffer_info(nullptr, sizeof(CompareStruct), "T{?:x:3xI:y:f:z:}", 1)))); return list; }); - m.def("buffer_to_dtype", [](py::buffer& buf) { return py::dtype(buf.request()); }); + m.def("buffer_to_dtype", [](py::buffer &buf) { return py::dtype(buf.request()); }); // test_scalar_conversion auto f_simple = [](SimpleStruct s) { return s.uint_ * 10; }; @@ -492,8 +606,9 @@ TEST_SUBMODULE(numpy_dtypes, m) { m.def("f_simple_pass_thru_vectorized", py::vectorize(f_simple_pass_thru)); // test_register_dtype - m.def("register_dtype", []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); }); + m.def("register_dtype", + []() { PYBIND11_NUMPY_DTYPE(SimpleStruct, bool_, uint_, float_, ldbl_); }); // test_str_leak - m.def("dtype_wrapper", [](py::object d) { return py::dtype::from_args(std::move(d)); }); + m.def("dtype_wrapper", [](const py::object &d) { return py::dtype::from_args(d); }); } diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index f56b776a..d10457ee 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- import re import pytest import env # noqa: F401 - from pybind11_tests import numpy_dtypes as m np = pytest.importorskip("numpy") @@ -16,7 +14,7 @@ def simple_dtype(): return np.dtype( { "names": ["bool_", "uint_", "float_", "ldbl_"], - "formats": ["?", "u4", "f4", "f{}".format(ld.itemsize)], + "formats": ["?", "u4", "f4", f"f{ld.itemsize}"], "offsets": [0, 4, 8, (16 if ld.alignment > 4 else 12)], } ) @@ -33,8 +31,8 @@ def dt_fmt(): e = "<" if byteorder == "little" else ">" return ( "{{'names':['bool_','uint_','float_','ldbl_']," - " 'formats':['?','" + e + "u4','" + e + "f4','" + e + "f{}']," - " 'offsets':[0,4,8,{}], 'itemsize':{}}}" + "'formats':['?','" + e + "u4','" + e + "f4','" + e + "f{}']," + "'offsets':[0,4,8,{}],'itemsize':{}}}" ) @@ -47,7 +45,7 @@ def simple_dtype_fmt(): def packed_dtype_fmt(): from sys import byteorder - return "[('bool_', '?'), ('uint_', '{e}u4'), ('float_', '{e}f4'), ('ldbl_', '{e}f{}')]".format( + return "[('bool_','?'),('uint_','{e}u4'),('float_','{e}f4'),('ldbl_','{e}f{}')]".format( np.dtype("longdouble").itemsize, e="<" if byteorder == "little" else ">" ) @@ -64,15 +62,21 @@ def partial_ld_offset(): def partial_dtype_fmt(): ld = np.dtype("longdouble") partial_ld_off = partial_ld_offset() - return dt_fmt().format(ld.itemsize, partial_ld_off, partial_ld_off + ld.itemsize) + partial_size = partial_ld_off + ld.itemsize + partial_end_padding = partial_size % np.dtype("uint64").alignment + return dt_fmt().format( + ld.itemsize, partial_ld_off, partial_size + partial_end_padding + ) def partial_nested_fmt(): ld = np.dtype("longdouble") partial_nested_off = 8 + 8 * (ld.alignment > 8) partial_ld_off = partial_ld_offset() - partial_nested_size = partial_nested_off * 2 + partial_ld_off + ld.itemsize - return "{{'names':['a'], 'formats':[{}], 'offsets':[{}], 'itemsize':{}}}".format( + partial_size = partial_ld_off + ld.itemsize + partial_end_padding = partial_size % np.dtype("uint64").alignment + partial_nested_size = partial_nested_off * 2 + partial_size + partial_end_padding + return "{{'names':['a'],'formats':[{}],'offsets':[{}],'itemsize':{}}}".format( partial_dtype_fmt(), partial_nested_off, partial_nested_size ) @@ -92,10 +96,12 @@ def test_format_descriptors(): ldbl_fmt = ("4x" if ld.alignment > 4 else "") + ld.char ss_fmt = "^T{?:bool_:3xI:uint_:f:float_:" + ldbl_fmt + ":ldbl_:}" dbl = np.dtype("double") + end_padding = ld.itemsize % np.dtype("uint64").alignment partial_fmt = ( "^T{?:bool_:3xI:uint_:f:float_:" + str(4 * (dbl.alignment > 4) + dbl.itemsize + 8 * (ld.alignment > 8)) - + "xg:ldbl_:}" + + "xg:ldbl_:" + + (str(end_padding) + "x}" if end_padding > 0 else "}") ) nested_extra = str(max(8, ld.alignment)) assert m.print_format_descriptors() == [ @@ -116,25 +122,21 @@ def test_dtype(simple_dtype): e = "<" if byteorder == "little" else ">" - assert m.print_dtypes() == [ + assert [x.replace(" ", "") for x in m.print_dtypes()] == [ simple_dtype_fmt(), packed_dtype_fmt(), - "[('a', {}), ('b', {})]".format(simple_dtype_fmt(), packed_dtype_fmt()), + f"[('a',{simple_dtype_fmt()}),('b',{packed_dtype_fmt()})]", partial_dtype_fmt(), partial_nested_fmt(), - "[('a', 'S3'), ('b', 'S3')]", + "[('a','S3'),('b','S3')]", ( - "{{'names':['a','b','c','d'], " - + "'formats':[('S4', (3,)),('" - + e - + "i4', (2,)),('u1', (3,)),('" - + e - + "f4', (4, 2))], " - + "'offsets':[0,12,20,24], 'itemsize':56}}" - ).format(e=e), - "[('e1', '" + e + "i8'), ('e2', 'u1')]", - "[('x', 'i1'), ('y', '" + e + "u8')]", - "[('cflt', '" + e + "c8'), ('cdbl', '" + e + "c16')]", + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ), + "[('e1','" + e + "i8'),('e2','u1')]", + "[('x','i1'),('y','" + e + "u8')]", + "[('cflt','" + e + "c8'),('cdbl','" + e + "c16')]", ] d1 = np.dtype( @@ -154,6 +156,7 @@ def test_dtype(simple_dtype): d1, np.dtype("uint32"), d2, + np.dtype("d"), ] assert m.test_dtype_methods() == [ @@ -169,6 +172,14 @@ def test_dtype(simple_dtype): np.zeros(1, m.trailing_padding_dtype()) ) + expected_chars = "bhilqBHILQefdgFDG?MmO" + assert m.test_dtype_kind() == list("iiiiiuuuuuffffcccbMmO") + assert m.test_dtype_char_() == list(expected_chars) + assert m.test_dtype_num() == [np.dtype(ch).num for ch in expected_chars] + assert m.test_dtype_byteorder() == [np.dtype(ch).byteorder for ch in expected_chars] + assert m.test_dtype_alignment() == [np.dtype(ch).alignment for ch in expected_chars] + assert m.test_dtype_flags() == [chr(np.dtype(ch).flags) for ch in expected_chars] + def test_recarray(simple_dtype, packed_dtype): elements = [(False, 0, 0.0, -0.0), (True, 1, 1.5, -2.5), (False, 2, 3.0, -5.0)] @@ -228,7 +239,7 @@ def test_recarray(simple_dtype, packed_dtype): ] arr = m.create_rec_partial(3) - assert str(arr.dtype) == partial_dtype_fmt() + assert str(arr.dtype).replace(" ", "") == partial_dtype_fmt() partial_dtype = arr.dtype assert "" not in arr.dtype.fields assert partial_dtype.itemsize > simple_dtype.itemsize @@ -236,7 +247,7 @@ def test_recarray(simple_dtype, packed_dtype): assert_equal(arr, elements, packed_dtype) arr = m.create_rec_partial_nested(3) - assert str(arr.dtype) == partial_nested_fmt() + assert str(arr.dtype).replace(" ", "") == partial_nested_fmt() assert "" not in arr.dtype.fields assert "" not in arr.dtype.fields["a"][0].fields assert arr.dtype.itemsize > partial_dtype.itemsize @@ -275,20 +286,18 @@ def test_array_array(): e = "<" if byteorder == "little" else ">" arr = m.create_array_array(3) - assert str(arr.dtype) == ( - "{{'names':['a','b','c','d'], " - + "'formats':[('S4', (3,)),('" - + e - + "i4', (2,)),('u1', (3,)),('{e}f4', (4, 2))], " - + "'offsets':[0,12,20,24], 'itemsize':56}}" - ).format(e=e) + assert str(arr.dtype).replace(" ", "") == ( + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ) assert m.print_array_array(arr) == [ "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1}," - + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001}," - + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001}," - + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", ] assert arr["a"].tolist() == [ [b"ABCD", b"KLMN", b"UVWX"], diff --git a/tests/test_numpy_vectorize.cpp b/tests/test_numpy_vectorize.cpp index 274b7558..dcc4c6ac 100644 --- a/tests/test_numpy_vectorize.cpp +++ b/tests/test_numpy_vectorize.cpp @@ -8,66 +8,80 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/numpy.h> +#include "pybind11_tests.h" + +#include <utility> + double my_func(int x, float y, double z) { py::print("my_func(x:int={}, y:float={:.0f}, z:float={:.0f})"_s.format(x, y, z)); - return (float) x*y*z; + return (float) x * y * z; } TEST_SUBMODULE(numpy_vectorize, m) { - try { py::module_::import("numpy"); } - catch (...) { return; } + try { + py::module_::import("numpy"); + } catch (const py::error_already_set &) { + return; + } // test_vectorize, test_docs, test_array_collapse // Vectorize all arguments of a function (though non-vector arguments are also allowed) m.def("vectorized_func", py::vectorize(my_func)); - // Vectorize a lambda function with a capture object (e.g. to exclude some arguments from the vectorization) - m.def("vectorized_func2", - [](py::array_t<int> x, py::array_t<float> y, float z) { - return py::vectorize([z](int x, float y) { return my_func(x, y, z); })(x, y); - } - ); + // Vectorize a lambda function with a capture object (e.g. to exclude some arguments from the + // vectorization) + m.def("vectorized_func2", [](py::array_t<int> x, py::array_t<float> y, float z) { + return py::vectorize([z](int x, float y) { return my_func(x, y, z); })(std::move(x), + std::move(y)); + }); // Vectorize a complex-valued function - m.def("vectorized_func3", py::vectorize( - [](std::complex<double> c) { return c * std::complex<double>(2.f); } - )); + m.def("vectorized_func3", + py::vectorize([](std::complex<double> c) { return c * std::complex<double>(2.f); })); // test_type_selection // NumPy function which only accepts specific data types - m.def("selective_func", [](py::array_t<int, py::array::c_style>) { return "Int branch taken."; }); - m.def("selective_func", [](py::array_t<float, py::array::c_style>) { return "Float branch taken."; }); - m.def("selective_func", [](py::array_t<std::complex<float>, py::array::c_style>) { return "Complex float branch taken."; }); - + // A lot of these no lints could be replaced with const refs, and probably should at some + // point. + m.def("selective_func", + [](const py::array_t<int, py::array::c_style> &) { return "Int branch taken."; }); + m.def("selective_func", + [](const py::array_t<float, py::array::c_style> &) { return "Float branch taken."; }); + m.def("selective_func", [](const py::array_t<std::complex<float>, py::array::c_style> &) { + return "Complex float branch taken."; + }); // test_passthrough_arguments - // Passthrough test: references and non-pod types should be automatically passed through (in the - // function definition below, only `b`, `d`, and `g` are vectorized): + // Passthrough test: references and non-pod types should be automatically passed through (in + // the function definition below, only `b`, `d`, and `g` are vectorized): struct NonPODClass { - NonPODClass(int v) : value{v} {} + explicit NonPODClass(int v) : value{v} {} int value; }; py::class_<NonPODClass>(m, "NonPODClass") .def(py::init<int>()) .def_readwrite("value", &NonPODClass::value); - m.def("vec_passthrough", py::vectorize( - [](double *a, double b, py::array_t<double> c, const int &d, int &e, NonPODClass f, const double g) { - return *a + b + c.at(0) + d + e + f.value + g; - } - )); + m.def("vec_passthrough", + py::vectorize([](const double *a, + double b, + // Changing this broke things + // NOLINTNEXTLINE(performance-unnecessary-value-param) + py::array_t<double> c, + const int &d, + int &e, + NonPODClass f, + const double g) { return *a + b + c.at(0) + d + e + f.value + g; })); // test_method_vectorization struct VectorizeTestClass { - VectorizeTestClass(int v) : value{v} {}; - float method(int x, float y) { return y + (float) (x + value); } + explicit VectorizeTestClass(int v) : value{v} {}; + float method(int x, float y) const { return y + (float) (x + value); } int value = 0; }; py::class_<VectorizeTestClass> vtc(m, "VectorizeTestClass"); - vtc .def(py::init<int>()) - .def_readwrite("value", &VectorizeTestClass::value); + vtc.def(py::init<int>()).def_readwrite("value", &VectorizeTestClass::value); // Automatic vectorizing of methods vtc.def("method", py::vectorize(&VectorizeTestClass::method)); @@ -78,16 +92,16 @@ TEST_SUBMODULE(numpy_vectorize, m) { .value("f_trivial", py::detail::broadcast_trivial::f_trivial) .value("c_trivial", py::detail::broadcast_trivial::c_trivial) .value("non_trivial", py::detail::broadcast_trivial::non_trivial); - m.def("vectorized_is_trivial", []( - py::array_t<int, py::array::forcecast> arg1, - py::array_t<float, py::array::forcecast> arg2, - py::array_t<double, py::array::forcecast> arg3 - ) { - py::ssize_t ndim; - std::vector<py::ssize_t> shape; - std::array<py::buffer_info, 3> buffers {{ arg1.request(), arg2.request(), arg3.request() }}; - return py::detail::broadcast(buffers, ndim, shape); - }); + m.def("vectorized_is_trivial", + [](const py::array_t<int, py::array::forcecast> &arg1, + const py::array_t<float, py::array::forcecast> &arg2, + const py::array_t<double, py::array::forcecast> &arg3) { + py::ssize_t ndim = 0; + std::vector<py::ssize_t> shape; + std::array<py::buffer_info, 3> buffers{ + {arg1.request(), arg2.request(), arg3.request()}}; + return py::detail::broadcast(buffers, ndim, shape); + }); - m.def("add_to", py::vectorize([](NonPODClass& x, int a) { x.value += a; })); + m.def("add_to", py::vectorize([](NonPODClass &x, int a) { x.value += a; })); } diff --git a/tests/test_numpy_vectorize.py b/tests/test_numpy_vectorize.py index 4e6b2d19..f1e8b625 100644 --- a/tests/test_numpy_vectorize.py +++ b/tests/test_numpy_vectorize.py @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- import pytest + from pybind11_tests import numpy_vectorize as m np = pytest.importorskip("numpy") @@ -149,7 +149,7 @@ def test_docs(doc): doc(m.vectorized_func) == """ vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object - """ # noqa: E501 line too long + """ ) diff --git a/tests/test_opaque_types.cpp b/tests/test_opaque_types.cpp index 5a234316..0386dba0 100644 --- a/tests/test_opaque_types.cpp +++ b/tests/test_opaque_types.cpp @@ -7,8 +7,10 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/stl.h> + +#include "pybind11_tests.h" + #include <vector> // IMPORTANT: Disable internal pybind11 translation mechanisms for STL data structures @@ -26,12 +28,13 @@ TEST_SUBMODULE(opaque_types, m) { .def(py::init<>()) .def("pop_back", &StringList::pop_back) /* There are multiple versions of push_back(), etc. Select the right ones. */ - .def("push_back", (void (StringList::*)(const std::string &)) &StringList::push_back) - .def("back", (std::string &(StringList::*)()) &StringList::back) + .def("push_back", (void(StringList::*)(const std::string &)) & StringList::push_back) + .def("back", (std::string & (StringList::*) ()) & StringList::back) .def("__len__", [](const StringList &v) { return v.size(); }) - .def("__iter__", [](StringList &v) { - return py::make_iterator(v.begin(), v.end()); - }, py::keep_alive<0, 1>()); + .def( + "__iter__", + [](StringList &v) { return py::make_iterator(v.begin(), v.end()); }, + py::keep_alive<0, 1>()); class ClassWithSTLVecProperty { public: @@ -44,9 +47,10 @@ TEST_SUBMODULE(opaque_types, m) { m.def("print_opaque_list", [](const StringList &l) { std::string ret = "Opaque list: ["; bool first = true; - for (auto entry : l) { - if (!first) + for (const auto &entry : l) { + if (!first) { ret += ", "; + } ret += entry; first = false; } diff --git a/tests/test_opaque_types.py b/tests/test_opaque_types.py index 77379463..5d4f2a1b 100644 --- a/tests/test_opaque_types.py +++ b/tests/test_opaque_types.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- import pytest -from pybind11_tests import opaque_types as m + from pybind11_tests import ConstructorStats, UserType +from pybind11_tests import opaque_types as m def test_string_list(): @@ -12,7 +12,7 @@ def test_string_list(): assert lst.back() == "Element 2" for i, k in enumerate(lst, start=1): - assert k == "Element {}".format(i) + assert k == f"Element {i}" lst.pop_back() assert m.print_opaque_list(lst) == "Opaque list: [Element 1]" @@ -39,7 +39,7 @@ def test_pointers(msg): 1. (arg0: capsule) -> int Invoked with: [1, 2, 3] - """ # noqa: E501 line too long + """ ) assert m.return_null_str() is None diff --git a/tests/test_operator_overloading.cpp b/tests/test_operator_overloading.cpp index 0a27bfd5..112a363b 100644 --- a/tests/test_operator_overloading.cpp +++ b/tests/test_operator_overloading.cpp @@ -7,21 +7,40 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/operators.h> +#include <pybind11/stl.h> + +#include "constructor_stats.h" +#include "pybind11_tests.h" + #include <functional> class Vector2 { public: Vector2(float x, float y) : x(x), y(y) { print_created(this, toString()); } Vector2(const Vector2 &v) : x(v.x), y(v.y) { print_copy_created(this); } - Vector2(Vector2 &&v) : x(v.x), y(v.y) { print_move_created(this); v.x = v.y = 0; } - Vector2 &operator=(const Vector2 &v) { x = v.x; y = v.y; print_copy_assigned(this); return *this; } - Vector2 &operator=(Vector2 &&v) { x = v.x; y = v.y; v.x = v.y = 0; print_move_assigned(this); return *this; } + Vector2(Vector2 &&v) noexcept : x(v.x), y(v.y) { + print_move_created(this); + v.x = v.y = 0; + } + Vector2 &operator=(const Vector2 &v) { + x = v.x; + y = v.y; + print_copy_assigned(this); + return *this; + } + Vector2 &operator=(Vector2 &&v) noexcept { + x = v.x; + y = v.y; + v.x = v.y = 0; + print_move_assigned(this); + return *this; + } ~Vector2() { print_destroyed(this); } - std::string toString() const { return "[" + std::to_string(x) + ", " + std::to_string(y) + "]"; } + std::string toString() const { + return "[" + std::to_string(x) + ", " + std::to_string(y) + "]"; + } Vector2 operator-() const { return Vector2(-x, -y); } Vector2 operator+(const Vector2 &v) const { return Vector2(x + v.x, y + v.y); } @@ -32,71 +51,100 @@ public: Vector2 operator/(float value) const { return Vector2(x / value, y / value); } Vector2 operator*(const Vector2 &v) const { return Vector2(x * v.x, y * v.y); } Vector2 operator/(const Vector2 &v) const { return Vector2(x / v.x, y / v.y); } - Vector2& operator+=(const Vector2 &v) { x += v.x; y += v.y; return *this; } - Vector2& operator-=(const Vector2 &v) { x -= v.x; y -= v.y; return *this; } - Vector2& operator*=(float v) { x *= v; y *= v; return *this; } - Vector2& operator/=(float v) { x /= v; y /= v; return *this; } - Vector2& operator*=(const Vector2 &v) { x *= v.x; y *= v.y; return *this; } - Vector2& operator/=(const Vector2 &v) { x /= v.x; y /= v.y; return *this; } + Vector2 &operator+=(const Vector2 &v) { + x += v.x; + y += v.y; + return *this; + } + Vector2 &operator-=(const Vector2 &v) { + x -= v.x; + y -= v.y; + return *this; + } + Vector2 &operator*=(float v) { + x *= v; + y *= v; + return *this; + } + Vector2 &operator/=(float v) { + x /= v; + y /= v; + return *this; + } + Vector2 &operator*=(const Vector2 &v) { + x *= v.x; + y *= v.y; + return *this; + } + Vector2 &operator/=(const Vector2 &v) { + x /= v.x; + y /= v.y; + return *this; + } friend Vector2 operator+(float f, const Vector2 &v) { return Vector2(f + v.x, f + v.y); } friend Vector2 operator-(float f, const Vector2 &v) { return Vector2(f - v.x, f - v.y); } friend Vector2 operator*(float f, const Vector2 &v) { return Vector2(f * v.x, f * v.y); } friend Vector2 operator/(float f, const Vector2 &v) { return Vector2(f / v.x, f / v.y); } - bool operator==(const Vector2 &v) const { - return x == v.x && y == v.y; - } - bool operator!=(const Vector2 &v) const { - return x != v.x || y != v.y; - } + bool operator==(const Vector2 &v) const { return x == v.x && y == v.y; } + bool operator!=(const Vector2 &v) const { return x != v.x || y != v.y; } + private: float x, y; }; -class C1 { }; -class C2 { }; +class C1 {}; +class C2 {}; int operator+(const C1 &, const C1 &) { return 11; } int operator+(const C2 &, const C2 &) { return 22; } int operator+(const C2 &, const C1 &) { return 21; } int operator+(const C1 &, const C2 &) { return 12; } +struct HashMe { + std::string member; +}; + +bool operator==(const HashMe &lhs, const HashMe &rhs) { return lhs.member == rhs.member; } + // Note: Specializing explicit within `namespace std { ... }` is done due to a // bug in GCC<7. If you are supporting compilers later than this, consider // specializing `using template<> struct std::hash<...>` in the global // namespace instead, per this recommendation: // https://en.cppreference.com/w/cpp/language/extending_std#Adding_template_specializations namespace std { - template<> - struct hash<Vector2> { - // Not a good hash function, but easy to test - size_t operator()(const Vector2 &) { return 4; } - }; +template <> +struct hash<Vector2> { + // Not a good hash function, but easy to test + size_t operator()(const Vector2 &) { return 4; } +}; + +// HashMe has a hash function in C++ but no `__hash__` for Python. +template <> +struct hash<HashMe> { + std::size_t operator()(const HashMe &selector) const { + return std::hash<std::string>()(selector.member); + } +}; } // namespace std // Not a good abs function, but easy to test. -std::string abs(const Vector2&) { - return "abs(Vector2)"; -} +std::string abs(const Vector2 &) { return "abs(Vector2)"; } -// MSVC warns about unknown pragmas, and warnings are errors. -#ifndef _MSC_VER - #pragma GCC diagnostic push - // clang 7.0.0 and Apple LLVM 10.0.1 introduce `-Wself-assign-overloaded` to - // `-Wall`, which is used here for overloading (e.g. `py::self += py::self `). - // Here, we suppress the warning using `#pragma diagnostic`. - // Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 - // TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). - #if defined(__APPLE__) && defined(__clang__) - #if (__clang_major__ >= 10) - #pragma GCC diagnostic ignored "-Wself-assign-overloaded" - #endif - #elif defined(__clang__) - #if (__clang_major__ >= 7) - #pragma GCC diagnostic ignored "-Wself-assign-overloaded" - #endif - #endif +// clang 7.0.0 and Apple LLVM 10.0.1 introduce `-Wself-assign-overloaded` to +// `-Wall`, which is used here for overloading (e.g. `py::self += py::self `). +// Here, we suppress the warning +// Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 +// TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). +#if defined(__APPLE__) && defined(__clang__) +# if (__clang_major__ >= 10) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") +# endif +#elif defined(__clang__) +# if (__clang_major__ >= 7) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") +# endif #endif TEST_SUBMODULE(operators, m) { @@ -130,46 +178,52 @@ TEST_SUBMODULE(operators, m) { .def(py::hash(py::self)) // N.B. See warning about usage of `py::detail::abs(py::self)` in // `operators.h`. - .def("__abs__", [](const Vector2& v) { return abs(v); }) - ; + .def("__abs__", [](const Vector2 &v) { return abs(v); }); m.attr("Vector") = m.attr("Vector2"); // test_operators_notimplemented // #393: need to return NotSupported to ensure correct arithmetic operator behavior - py::class_<C1>(m, "C1") - .def(py::init<>()) - .def(py::self + py::self); + py::class_<C1>(m, "C1").def(py::init<>()).def(py::self + py::self); py::class_<C2>(m, "C2") .def(py::init<>()) .def(py::self + py::self) - .def("__add__", [](const C2& c2, const C1& c1) { return c2 + c1; }) - .def("__radd__", [](const C2& c2, const C1& c1) { return c1 + c2; }); + .def("__add__", [](const C2 &c2, const C1 &c1) { return c2 + c1; }) + .def("__radd__", [](const C2 &c2, const C1 &c1) { return c1 + c2; }); // test_nested // #328: first member in a class can't be used in operators - struct NestABase { int value = -2; }; + struct NestABase { + int value = -2; + }; py::class_<NestABase>(m, "NestABase") .def(py::init<>()) .def_readwrite("value", &NestABase::value); struct NestA : NestABase { int value = 3; - NestA& operator+=(int i) { value += i; return *this; } + NestA &operator+=(int i) { + value += i; + return *this; + } }; py::class_<NestA>(m, "NestA") .def(py::init<>()) .def(py::self += int()) - .def("as_base", [](NestA &a) -> NestABase& { - return (NestABase&) a; - }, py::return_value_policy::reference_internal); + .def( + "as_base", + [](NestA &a) -> NestABase & { return (NestABase &) a; }, + py::return_value_policy::reference_internal); m.def("get_NestA", [](const NestA &a) { return a.value; }); struct NestB { NestA a; int value = 4; - NestB& operator-=(int i) { value -= i; return *this; } + NestB &operator-=(int i) { + value -= i; + return *this; + } }; py::class_<NestB>(m, "NestB") .def(py::init<>()) @@ -180,7 +234,10 @@ TEST_SUBMODULE(operators, m) { struct NestC { NestB b; int value = 5; - NestC& operator*=(int i) { value *= i; return *this; } + NestC &operator*=(int i) { + value *= i; + return *this; + } }; py::class_<NestC>(m, "NestC") .def(py::init<>()) @@ -188,16 +245,15 @@ TEST_SUBMODULE(operators, m) { .def_readwrite("b", &NestC::b); m.def("get_NestC", [](const NestC &c) { return c.value; }); - // test_overriding_eq_reset_hash // #2191 Overriding __eq__ should set __hash__ to None struct Comparable { int value; - bool operator==(const Comparable& rhs) const {return value == rhs.value;} + bool operator==(const Comparable &rhs) const { return value == rhs.value; } }; struct Hashable : Comparable { - explicit Hashable(int value): Comparable{value}{}; + explicit Hashable(int value) : Comparable{value} {}; size_t hash() const { return static_cast<size_t>(value); } }; @@ -205,9 +261,7 @@ TEST_SUBMODULE(operators, m) { using Hashable::Hashable; }; - py::class_<Comparable>(m, "Comparable") - .def(py::init<int>()) - .def(py::self == py::self); + py::class_<Comparable>(m, "Comparable").def(py::init<int>()).def(py::self == py::self); py::class_<Hashable>(m, "Hashable") .def(py::init<int>()) @@ -219,8 +273,9 @@ TEST_SUBMODULE(operators, m) { .def("__hash__", &Hashable::hash) .def(py::init<int>()) .def(py::self == py::self); -} -#ifndef _MSC_VER - #pragma GCC diagnostic pop -#endif + // define __eq__ but not __hash__ + py::class_<HashMe>(m, "HashMe").def(py::self == py::self); + + m.def("get_unhashable_HashMe_set", []() { return std::unordered_set<HashMe>{{"one"}}; }); +} diff --git a/tests/test_operator_overloading.py b/tests/test_operator_overloading.py index 5dbfb32c..9fde305a 100644 --- a/tests/test_operator_overloading.py +++ b/tests/test_operator_overloading.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- import pytest -from pybind11_tests import operators as m + from pybind11_tests import ConstructorStats +from pybind11_tests import operators as m def test_operator_overloading(): @@ -130,12 +130,12 @@ def test_nested(): def test_overriding_eq_reset_hash(): - assert m.Comparable(15) is not m.Comparable(15) assert m.Comparable(15) == m.Comparable(15) - with pytest.raises(TypeError): - hash(m.Comparable(15)) # TypeError: unhashable type: 'm.Comparable' + with pytest.raises(TypeError) as excinfo: + hash(m.Comparable(15)) + assert str(excinfo.value).startswith("unhashable type:") for hashable in (m.Hashable, m.Hashable2): assert hashable(15) is not hashable(15) @@ -143,3 +143,9 @@ def test_overriding_eq_reset_hash(): assert hash(hashable(15)) == 15 assert hash(hashable(15)) == hash(hashable(15)) + + +def test_return_set_of_unhashable(): + with pytest.raises(TypeError) as excinfo: + m.get_unhashable_HashMe_set() + assert str(excinfo.value.__cause__).startswith("unhashable type:") diff --git a/tests/test_pickling.cpp b/tests/test_pickling.cpp index 1a48595a..e154bc48 100644 --- a/tests/test_pickling.cpp +++ b/tests/test_pickling.cpp @@ -2,6 +2,7 @@ tests/test_pickling.cpp -- pickle support Copyright (c) 2016 Wenzel Jakob <wenzel.jakob@epfl.ch> + Copyright (c) 2021 The Pybind Development Team. All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. @@ -9,17 +10,70 @@ #include "pybind11_tests.h" +#include <memory> +#include <stdexcept> +#include <utility> + +namespace exercise_trampoline { + +struct SimpleBase { + int num = 0; + virtual ~SimpleBase() = default; + + // For compatibility with old clang versions: + SimpleBase() = default; + SimpleBase(const SimpleBase &) = default; +}; + +struct SimpleBaseTrampoline : SimpleBase {}; + +struct SimpleCppDerived : SimpleBase {}; + +void wrap(py::module m) { + py::class_<SimpleBase, SimpleBaseTrampoline>(m, "SimpleBase") + .def(py::init<>()) + .def_readwrite("num", &SimpleBase::num) + .def(py::pickle( + [](const py::object &self) { + py::dict d; + if (py::hasattr(self, "__dict__")) { + d = self.attr("__dict__"); + } + return py::make_tuple(self.attr("num"), d); + }, + [](const py::tuple &t) { + if (t.size() != 2) { + throw std::runtime_error("Invalid state!"); + } + auto cpp_state = std::unique_ptr<SimpleBase>(new SimpleBaseTrampoline); + cpp_state->num = t[0].cast<int>(); + auto py_state = t[1].cast<py::dict>(); + return std::make_pair(std::move(cpp_state), py_state); + })); + + m.def("make_SimpleCppDerivedAsBase", + []() { return std::unique_ptr<SimpleBase>(new SimpleCppDerived); }); + m.def("check_dynamic_cast_SimpleCppDerived", [](const SimpleBase *base_ptr) { + return dynamic_cast<const SimpleCppDerived *>(base_ptr) != nullptr; + }); +} + +} // namespace exercise_trampoline + TEST_SUBMODULE(pickling, m) { + m.def("simple_callable", []() { return 20220426; }); + // test_roundtrip class Pickleable { public: - Pickleable(const std::string &value) : m_value(value) { } + explicit Pickleable(const std::string &value) : m_value(value) {} const std::string &value() const { return m_value; } void setExtra1(int extra1) { m_extra1 = extra1; } void setExtra2(int extra2) { m_extra2 = extra2; } int extra1() const { return m_extra1; } int extra2() const { return m_extra2; } + private: std::string m_value; int m_extra1 = 0; @@ -32,8 +86,7 @@ TEST_SUBMODULE(pickling, m) { }; py::class_<Pickleable> pyPickleable(m, "Pickleable"); - pyPickleable - .def(py::init<std::string>()) + pyPickleable.def(py::init<std::string>()) .def("value", &Pickleable::value) .def("extra1", &Pickleable::extra1) .def("extra2", &Pickleable::extra2) @@ -46,17 +99,17 @@ TEST_SUBMODULE(pickling, m) { return py::make_tuple(p.value(), p.extra1(), p.extra2()); }); ignoreOldStyleInitWarnings([&pyPickleable]() { - pyPickleable - .def("__setstate__", [](Pickleable &p, py::tuple t) { - if (t.size() != 3) - throw std::runtime_error("Invalid state!"); - /* Invoke the constructor (need to use in-place version) */ - new (&p) Pickleable(t[0].cast<std::string>()); + pyPickleable.def("__setstate__", [](Pickleable &p, const py::tuple &t) { + if (t.size() != 3) { + throw std::runtime_error("Invalid state!"); + } + /* Invoke the constructor (need to use in-place version) */ + new (&p) Pickleable(t[0].cast<std::string>()); - /* Assign any additional state */ - p.setExtra1(t[1].cast<int>()); - p.setExtra2(t[2].cast<int>()); - }); + /* Assign any additional state */ + p.setExtra1(t[1].cast<int>()); + p.setExtra2(t[2].cast<int>()); + }); }); py::class_<PickleableNew, Pickleable>(m, "PickleableNew") @@ -65,22 +118,22 @@ TEST_SUBMODULE(pickling, m) { [](const PickleableNew &p) { return py::make_tuple(p.value(), p.extra1(), p.extra2()); }, - [](py::tuple t) { - if (t.size() != 3) + [](const py::tuple &t) { + if (t.size() != 3) { throw std::runtime_error("Invalid state!"); + } auto p = PickleableNew(t[0].cast<std::string>()); p.setExtra1(t[1].cast<int>()); p.setExtra2(t[2].cast<int>()); return p; - } - )); + })); #if !defined(PYPY_VERSION) // test_roundtrip_with_dict class PickleableWithDict { public: - PickleableWithDict(const std::string &value) : value(value) { } + explicit PickleableWithDict(const std::string &value) : value(value) {} std::string value; int extra; @@ -91,48 +144,51 @@ TEST_SUBMODULE(pickling, m) { using PickleableWithDict::PickleableWithDict; }; - py::class_<PickleableWithDict> pyPickleableWithDict(m, "PickleableWithDict", py::dynamic_attr()); - pyPickleableWithDict - .def(py::init<std::string>()) + py::class_<PickleableWithDict> pyPickleableWithDict( + m, "PickleableWithDict", py::dynamic_attr()); + pyPickleableWithDict.def(py::init<std::string>()) .def_readwrite("value", &PickleableWithDict::value) .def_readwrite("extra", &PickleableWithDict::extra) - .def("__getstate__", [](py::object self) { + .def("__getstate__", [](const py::object &self) { /* Also include __dict__ in state */ return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__")); }); ignoreOldStyleInitWarnings([&pyPickleableWithDict]() { - pyPickleableWithDict - .def("__setstate__", [](py::object self, py::tuple t) { - if (t.size() != 3) - throw std::runtime_error("Invalid state!"); - /* Cast and construct */ - auto& p = self.cast<PickleableWithDict&>(); - new (&p) PickleableWithDict(t[0].cast<std::string>()); + pyPickleableWithDict.def("__setstate__", [](const py::object &self, const py::tuple &t) { + if (t.size() != 3) { + throw std::runtime_error("Invalid state!"); + } + /* Cast and construct */ + auto &p = self.cast<PickleableWithDict &>(); + new (&p) PickleableWithDict(t[0].cast<std::string>()); - /* Assign C++ state */ - p.extra = t[1].cast<int>(); + /* Assign C++ state */ + p.extra = t[1].cast<int>(); - /* Assign Python state */ - self.attr("__dict__") = t[2]; - }); + /* Assign Python state */ + self.attr("__dict__") = t[2]; + }); }); py::class_<PickleableWithDictNew, PickleableWithDict>(m, "PickleableWithDictNew") .def(py::init<std::string>()) .def(py::pickle( - [](py::object self) { - return py::make_tuple(self.attr("value"), self.attr("extra"), self.attr("__dict__")); + [](const py::object &self) { + return py::make_tuple( + self.attr("value"), self.attr("extra"), self.attr("__dict__")); }, [](const py::tuple &t) { - if (t.size() != 3) + if (t.size() != 3) { throw std::runtime_error("Invalid state!"); + } auto cpp_state = PickleableWithDictNew(t[0].cast<std::string>()); cpp_state.extra = t[1].cast<int>(); auto py_state = t[2].cast<py::dict>(); return std::make_pair(cpp_state, py_state); - } - )); + })); #endif + + exercise_trampoline::wrap(m); } diff --git a/tests/test_pickling.py b/tests/test_pickling.py index 6b27a73a..12361a66 100644 --- a/tests/test_pickling.py +++ b/tests/test_pickling.py @@ -1,14 +1,24 @@ -# -*- coding: utf-8 -*- -import pytest +import pickle +import re -import env # noqa: F401 +import pytest +import env from pybind11_tests import pickling as m -try: - import cPickle as pickle # Use cPickle on Python 2.7 -except ImportError: - import pickle + +def test_pickle_simple_callable(): + assert m.simple_callable() == 20220426 + if env.PYPY: + serialized = pickle.dumps(m.simple_callable) + deserialized = pickle.loads(serialized) + assert deserialized() == 20220426 + else: + # To document broken behavior: currently it fails universally with + # all C Python versions. + with pytest.raises(TypeError) as excinfo: + pickle.dumps(m.simple_callable) + assert re.search("can.*t pickle .*PyCapsule.* object", str(excinfo.value)) @pytest.mark.parametrize("cls_name", ["Pickleable", "PickleableNew"]) @@ -45,3 +55,39 @@ def test_enum_pickle(): data = pickle.dumps(e.EOne, 2) assert e.EOne == pickle.loads(data) + + +# +# exercise_trampoline +# +class SimplePyDerived(m.SimpleBase): + pass + + +def test_roundtrip_simple_py_derived(): + p = SimplePyDerived() + p.num = 202 + p.stored_in_dict = 303 + data = pickle.dumps(p, pickle.HIGHEST_PROTOCOL) + p2 = pickle.loads(data) + assert isinstance(p2, SimplePyDerived) + assert p2.num == 202 + assert p2.stored_in_dict == 303 + + +def test_roundtrip_simple_cpp_derived(): + p = m.make_SimpleCppDerivedAsBase() + assert m.check_dynamic_cast_SimpleCppDerived(p) + p.num = 404 + if not env.PYPY: + # To ensure that this unit test is not accidentally invalidated. + with pytest.raises(AttributeError): + # Mimics the `setstate` C++ implementation. + setattr(p, "__dict__", {}) # noqa: B010 + data = pickle.dumps(p, pickle.HIGHEST_PROTOCOL) + p2 = pickle.loads(data) + assert isinstance(p2, m.SimpleBase) + assert p2.num == 404 + # Issue #3062: pickleable base C++ classes can incur object slicing + # if derived typeid is not registered with pybind11 + assert not m.check_dynamic_cast_SimpleCppDerived(p2) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 709611d2..b4ee6428 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -9,15 +9,121 @@ #include "pybind11_tests.h" +#include <utility> + +namespace external { +namespace detail { +bool check(PyObject *o) { return PyFloat_Check(o) != 0; } + +PyObject *conv(PyObject *o) { + PyObject *ret = nullptr; + if (PyLong_Check(o)) { + double v = PyLong_AsDouble(o); + if (!(v == -1.0 && PyErr_Occurred())) { + ret = PyFloat_FromDouble(v); + } + } else { + PyErr_SetString(PyExc_TypeError, "Unexpected type"); + } + return ret; +} + +PyObject *default_constructed() { return PyFloat_FromDouble(0.0); } +} // namespace detail +class float_ : public py::object { + PYBIND11_OBJECT_CVT(float_, py::object, external::detail::check, external::detail::conv) + + float_() : py::object(external::detail::default_constructed(), stolen_t{}) {} + + double get_value() const { return PyFloat_AsDouble(this->ptr()); } +}; +} // namespace external + +namespace implicit_conversion_from_0_to_handle { +// Uncomment to trigger compiler error. Note: Before PR #4008 this used to compile successfully. +// void expected_to_trigger_compiler_error() { py::handle(0); } +} // namespace implicit_conversion_from_0_to_handle + +// Used to validate systematically that PR #4008 does/did NOT change the behavior. +void pure_compile_tests_for_handle_from_PyObject_pointers() { + { + PyObject *ptr = Py_None; + py::handle{ptr}; + } + { + PyObject *const ptr = Py_None; + py::handle{ptr}; + } + // Uncomment to trigger compiler errors. + // PyObject const * ptr = Py_None; py::handle{ptr}; + // PyObject const *const ptr = Py_None; py::handle{ptr}; + // PyObject volatile * ptr = Py_None; py::handle{ptr}; + // PyObject volatile *const ptr = Py_None; py::handle{ptr}; + // PyObject const volatile * ptr = Py_None; py::handle{ptr}; + // PyObject const volatile *const ptr = Py_None; py::handle{ptr}; +} + +namespace handle_from_move_only_type_with_operator_PyObject { + +// Reduced from +// https://github.com/pytorch/pytorch/blob/279634f384662b7c3a9f8bf7ccc3a6afd2f05657/torch/csrc/utils/object_ptr.h +struct operator_ncnst { + operator_ncnst() = default; + operator_ncnst(operator_ncnst &&) = default; + operator PyObject *() /* */ { return Py_None; } // NOLINT(google-explicit-constructor) +}; + +struct operator_const { + operator_const() = default; + operator_const(operator_const &&) = default; + operator PyObject *() const { return Py_None; } // NOLINT(google-explicit-constructor) +}; + +bool from_ncnst() { + operator_ncnst obj; + auto h = py::handle(obj); // Critical part of test: does this compile? + return h.ptr() == Py_None; // Just something. +} + +bool from_const() { + operator_const obj; + auto h = py::handle(obj); // Critical part of test: does this compile? + return h.ptr() == Py_None; // Just something. +} + +void m_defs(py::module_ &m) { + m.def("handle_from_move_only_type_with_operator_PyObject_ncnst", from_ncnst); + m.def("handle_from_move_only_type_with_operator_PyObject_const", from_const); +} + +} // namespace handle_from_move_only_type_with_operator_PyObject TEST_SUBMODULE(pytypes, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + + handle_from_move_only_type_with_operator_PyObject::m_defs(m); + + // test_bool + m.def("get_bool", [] { return py::bool_(false); }); // test_int - m.def("get_int", []{return py::int_(0);}); + m.def("get_int", [] { return py::int_(0); }); // test_iterator - m.def("get_iterator", []{return py::iterator();}); + m.def("get_iterator", [] { return py::iterator(); }); // test_iterable - m.def("get_iterable", []{return py::iterable();}); + m.def("get_iterable", [] { return py::iterable(); }); + m.def("get_frozenset_from_iterable", + [](const py::iterable &iter) { return py::frozenset(iter); }); + m.def("get_list_from_iterable", [](const py::iterable &iter) { return py::list(iter); }); + m.def("get_set_from_iterable", [](const py::iterable &iter) { return py::set(iter); }); + m.def("get_tuple_from_iterable", [](const py::iterable &iter) { return py::tuple(iter); }); + // test_float + m.def("get_float", [] { return py::float_(0.0f); }); // test_list + m.def("list_no_args", []() { return py::list{}; }); + m.def("list_ssize_t", []() { return py::list{(py::ssize_t) 0}; }); + m.def("list_size_t", []() { return py::list{(py::size_t) 0}; }); + m.def("list_insert_ssize_t", [](py::list *l) { return l->insert((py::ssize_t) 1, 83); }); + m.def("list_insert_size_t", [](py::list *l) { return l->insert((py::size_t) 3, 57); }); m.def("get_list", []() { py::list list; list.append("value"); @@ -27,18 +133,17 @@ TEST_SUBMODULE(pytypes, m) { list.insert(2, "inserted-2"); return list; }); - m.def("print_list", [](py::list list) { + m.def("print_list", [](const py::list &list) { int index = 0; - for (auto item : list) + for (auto item : list) { py::print("list item {}: {}"_s.format(index++, item)); + } }); // test_none - m.def("get_none", []{return py::none();}); - m.def("print_none", [](py::none none) { - py::print("none: {}"_s.format(none)); - }); + m.def("get_none", [] { return py::none(); }); + m.def("print_none", [](const py::none &none) { py::print("none: {}"_s.format(none)); }); - // test_set + // test_set, test_frozenset m.def("get_set", []() { py::set set; set.add(py::str("key1")); @@ -46,58 +151,106 @@ TEST_SUBMODULE(pytypes, m) { set.add(std::string("key3")); return set; }); - m.def("print_set", [](py::set set) { - for (auto item : set) - py::print("key:", item); - }); - m.def("set_contains", [](py::set set, py::object key) { - return set.contains(key); + m.def("get_frozenset", []() { + py::set set; + set.add(py::str("key1")); + set.add("key2"); + set.add(std::string("key3")); + return py::frozenset(set); }); - m.def("set_contains", [](py::set set, const char* key) { - return set.contains(key); + m.def("print_anyset", [](const py::anyset &set) { + for (auto item : set) { + py::print("key:", item); + } }); + m.def("anyset_size", [](const py::anyset &set) { return set.size(); }); + m.def("anyset_empty", [](const py::anyset &set) { return set.empty(); }); + m.def("anyset_contains", + [](const py::anyset &set, const py::object &key) { return set.contains(key); }); + m.def("anyset_contains", + [](const py::anyset &set, const char *key) { return set.contains(key); }); + m.def("set_add", [](py::set &set, const py::object &key) { set.add(key); }); + m.def("set_clear", [](py::set &set) { set.clear(); }); // test_dict - m.def("get_dict", []() { return py::dict("key"_a="value"); }); - m.def("print_dict", [](py::dict dict) { - for (auto item : dict) + m.def("get_dict", []() { return py::dict("key"_a = "value"); }); + m.def("print_dict", [](const py::dict &dict) { + for (auto item : dict) { py::print("key: {}, value={}"_s.format(item.first, item.second)); + } }); m.def("dict_keyword_constructor", []() { - auto d1 = py::dict("x"_a=1, "y"_a=2); - auto d2 = py::dict("z"_a=3, **d1); + auto d1 = py::dict("x"_a = 1, "y"_a = 2); + auto d2 = py::dict("z"_a = 3, **d1); return d2; }); - m.def("dict_contains", [](py::dict dict, py::object val) { - return dict.contains(val); - }); - m.def("dict_contains", [](py::dict dict, const char* val) { - return dict.contains(val); + m.def("dict_contains", + [](const py::dict &dict, const py::object &val) { return dict.contains(val); }); + m.def("dict_contains", + [](const py::dict &dict, const char *val) { return dict.contains(val); }); + + // test_tuple + m.def("tuple_no_args", []() { return py::tuple{}; }); + m.def("tuple_ssize_t", []() { return py::tuple{(py::ssize_t) 0}; }); + m.def("tuple_size_t", []() { return py::tuple{(py::size_t) 0}; }); + m.def("get_tuple", []() { return py::make_tuple(42, py::none(), "spam"); }); + + // test_simple_namespace + m.def("get_simple_namespace", []() { + auto ns = py::module_::import("types").attr("SimpleNamespace")( + "attr"_a = 42, "x"_a = "foo", "wrong"_a = 1); + py::delattr(ns, "wrong"); + py::setattr(ns, "right", py::int_(2)); + return ns; }); // test_str + m.def("str_from_char_ssize_t", []() { return py::str{"red", (py::ssize_t) 3}; }); + m.def("str_from_char_size_t", []() { return py::str{"blue", (py::size_t) 4}; }); m.def("str_from_string", []() { return py::str(std::string("baz")); }); + m.def("str_from_std_string_input", [](const std::string &stri) { return py::str(stri); }); + m.def("str_from_cstr_input", [](const char *c_str) { return py::str(c_str); }); m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); - m.def("str_from_object", [](const py::object& obj) { return py::str(obj); }); - m.def("repr_from_object", [](const py::object& obj) { return py::repr(obj); }); + m.def("str_from_bytes_input", + [](const py::bytes &encoded_str) { return py::str(encoded_str); }); + + m.def("str_from_object", [](const py::object &obj) { return py::str(obj); }); + m.def("repr_from_object", [](const py::object &obj) { return py::repr(obj); }); m.def("str_from_handle", [](py::handle h) { return py::str(h); }); + m.def("str_from_string_from_str", + [](const py::str &obj) { return py::str(static_cast<std::string>(obj)); }); m.def("str_format", []() { auto s1 = "{} + {} = {}"_s.format(1, 2, 3); - auto s2 = "{a} + {b} = {c}"_s.format("a"_a=1, "b"_a=2, "c"_a=3); + auto s2 = "{a} + {b} = {c}"_s.format("a"_a = 1, "b"_a = 2, "c"_a = 3); return py::make_tuple(s1, s2); }); // test_bytes + m.def("bytes_from_char_ssize_t", []() { return py::bytes{"green", (py::ssize_t) 5}; }); + m.def("bytes_from_char_size_t", []() { return py::bytes{"purple", (py::size_t) 6}; }); m.def("bytes_from_string", []() { return py::bytes(std::string("foo")); }); m.def("bytes_from_str", []() { return py::bytes(py::str("bar", 3)); }); + // test bytearray + m.def("bytearray_from_char_ssize_t", []() { return py::bytearray{"$%", (py::ssize_t) 2}; }); + m.def("bytearray_from_char_size_t", []() { return py::bytearray{"@$!", (py::size_t) 3}; }); + m.def("bytearray_from_string", []() { return py::bytearray(std::string("foo")); }); + m.def("bytearray_size", []() { return py::bytearray("foo").size(); }); + // test_capsule m.def("return_capsule_with_destructor", []() { py::print("creating capsule"); - return py::capsule([]() { - py::print("destructing capsule"); - }); + return py::capsule([]() { py::print("destructing capsule"); }); + }); + + m.def("return_renamed_capsule_with_destructor", []() { + py::print("creating capsule"); + auto cap = py::capsule([]() { py::print("destructing capsule"); }); + static const char *capsule_name = "test_name1"; + py::print("renaming capsule"); + cap.set_name(capsule_name); + return cap; }); m.def("return_capsule_with_destructor_2", []() { @@ -107,33 +260,59 @@ TEST_SUBMODULE(pytypes, m) { }); }); + m.def("return_capsule_with_destructor_3", []() { + py::print("creating capsule"); + auto cap = py::capsule((void *) 1233, "oname", [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + py::print("original name: {}"_s.format(cap.name())); + return cap; + }); + + m.def("return_renamed_capsule_with_destructor_2", []() { + py::print("creating capsule"); + auto cap = py::capsule((void *) 1234, [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + static const char *capsule_name = "test_name2"; + py::print("renaming capsule"); + cap.set_name(capsule_name); + return cap; + }); + m.def("return_capsule_with_name_and_destructor", []() { auto capsule = py::capsule((void *) 12345, "pointer type description", [](PyObject *ptr) { if (ptr) { - auto name = PyCapsule_GetName(ptr); + const auto *name = PyCapsule_GetName(ptr); py::print("destructing capsule ({}, '{}')"_s.format( - (size_t) PyCapsule_GetPointer(ptr, name), name - )); + (size_t) PyCapsule_GetPointer(ptr, name), name)); } }); capsule.set_pointer((void *) 1234); // Using get_pointer<T>() - void* contents1 = static_cast<void*>(capsule); - void* contents2 = capsule.get_pointer(); - void* contents3 = capsule.get_pointer<void>(); + void *contents1 = static_cast<void *>(capsule); + void *contents2 = capsule.get_pointer(); + void *contents3 = capsule.get_pointer<void>(); auto result1 = reinterpret_cast<size_t>(contents1); auto result2 = reinterpret_cast<size_t>(contents2); auto result3 = reinterpret_cast<size_t>(contents3); - py::print("created capsule ({}, '{}')"_s.format(result1 & result2 & result3, capsule.name())); + py::print( + "created capsule ({}, '{}')"_s.format(result1 & result2 & result3, capsule.name())); return capsule; }); + m.def("return_capsule_with_explicit_nullptr_dtor", []() { + py::print("creating capsule with explicit nullptr dtor"); + return py::capsule(reinterpret_cast<void *>(1234), + static_cast<void (*)(void *)>(nullptr)); // PR #4221 + }); + // test_accessors - m.def("accessor_api", [](py::object o) { + m.def("accessor_api", [](const py::object &o) { auto d = py::dict(); d["basic_attr"] = o.attr("basic_attr"); @@ -174,7 +353,7 @@ TEST_SUBMODULE(pytypes, m) { return d; }); - m.def("tuple_accessor", [](py::tuple existing_t) { + m.def("tuple_accessor", [](const py::tuple &existing_t) { try { existing_t[0] = 1; } catch (const py::error_already_set &) { @@ -206,69 +385,118 @@ TEST_SUBMODULE(pytypes, m) { return d; }); + m.def("accessor_moves", []() { // See PR #3970 + py::list return_list; +#ifdef PYBIND11_HANDLE_REF_DEBUG + py::int_ py_int_0(0); + py::int_ py_int_42(42); + py::str py_str_count("count"); + + auto tup = py::make_tuple(0); + + py::sequence seq(tup); + + py::list lst; + lst.append(0); + +# define PYBIND11_LOCAL_DEF(...) \ + { \ + std::size_t inc_refs = py::handle::inc_ref_counter(); \ + __VA_ARGS__; \ + inc_refs = py::handle::inc_ref_counter() - inc_refs; \ + return_list.append(inc_refs); \ + } + + PYBIND11_LOCAL_DEF(tup[py_int_0]) // l-value (to have a control) + PYBIND11_LOCAL_DEF(tup[py::int_(0)]) // r-value + + PYBIND11_LOCAL_DEF(tup.attr(py_str_count)) // l-value + PYBIND11_LOCAL_DEF(tup.attr(py::str("count"))) // r-value + + PYBIND11_LOCAL_DEF(seq[py_int_0]) // l-value + PYBIND11_LOCAL_DEF(seq[py::int_(0)]) // r-value + + PYBIND11_LOCAL_DEF(seq.attr(py_str_count)) // l-value + PYBIND11_LOCAL_DEF(seq.attr(py::str("count"))) // r-value + + PYBIND11_LOCAL_DEF(lst[py_int_0]) // l-value + PYBIND11_LOCAL_DEF(lst[py::int_(0)]) // r-value + + PYBIND11_LOCAL_DEF(lst.attr(py_str_count)) // l-value + PYBIND11_LOCAL_DEF(lst.attr(py::str("count"))) // r-value + + auto lst_acc = lst[py::int_(0)]; + lst_acc = py::int_(42); // Detaches lst_acc from lst. + PYBIND11_LOCAL_DEF(lst_acc = py_int_42) // l-value + PYBIND11_LOCAL_DEF(lst_acc = py::int_(42)) // r-value +# undef PYBIND11_LOCAL_DEF +#endif + return return_list; + }); + // test_constructors m.def("default_constructors", []() { - return py::dict( - "bytes"_a=py::bytes(), - "str"_a=py::str(), - "bool"_a=py::bool_(), - "int"_a=py::int_(), - "float"_a=py::float_(), - "tuple"_a=py::tuple(), - "list"_a=py::list(), - "dict"_a=py::dict(), - "set"_a=py::set() - ); - }); - - m.def("converting_constructors", [](py::dict d) { - return py::dict( - "bytes"_a=py::bytes(d["bytes"]), - "str"_a=py::str(d["str"]), - "bool"_a=py::bool_(d["bool"]), - "int"_a=py::int_(d["int"]), - "float"_a=py::float_(d["float"]), - "tuple"_a=py::tuple(d["tuple"]), - "list"_a=py::list(d["list"]), - "dict"_a=py::dict(d["dict"]), - "set"_a=py::set(d["set"]), - "memoryview"_a=py::memoryview(d["memoryview"]) - ); - }); - - m.def("cast_functions", [](py::dict d) { + return py::dict("bytes"_a = py::bytes(), + "bytearray"_a = py::bytearray(), + "str"_a = py::str(), + "bool"_a = py::bool_(), + "int"_a = py::int_(), + "float"_a = py::float_(), + "tuple"_a = py::tuple(), + "list"_a = py::list(), + "dict"_a = py::dict(), + "set"_a = py::set()); + }); + + m.def("converting_constructors", [](const py::dict &d) { + return py::dict("bytes"_a = py::bytes(d["bytes"]), + "bytearray"_a = py::bytearray(d["bytearray"]), + "str"_a = py::str(d["str"]), + "bool"_a = py::bool_(d["bool"]), + "int"_a = py::int_(d["int"]), + "float"_a = py::float_(d["float"]), + "tuple"_a = py::tuple(d["tuple"]), + "list"_a = py::list(d["list"]), + "dict"_a = py::dict(d["dict"]), + "set"_a = py::set(d["set"]), + "frozenset"_a = py::frozenset(d["frozenset"]), + "memoryview"_a = py::memoryview(d["memoryview"])); + }); + + m.def("cast_functions", [](const py::dict &d) { // When converting between Python types, obj.cast<T>() should be the same as T(obj) - return py::dict( - "bytes"_a=d["bytes"].cast<py::bytes>(), - "str"_a=d["str"].cast<py::str>(), - "bool"_a=d["bool"].cast<py::bool_>(), - "int"_a=d["int"].cast<py::int_>(), - "float"_a=d["float"].cast<py::float_>(), - "tuple"_a=d["tuple"].cast<py::tuple>(), - "list"_a=d["list"].cast<py::list>(), - "dict"_a=d["dict"].cast<py::dict>(), - "set"_a=d["set"].cast<py::set>(), - "memoryview"_a=d["memoryview"].cast<py::memoryview>() - ); - }); - - m.def("convert_to_pybind11_str", [](py::object o) { return py::str(o); }); - - m.def("nonconverting_constructor", [](std::string type, py::object value, bool move) -> py::object { - if (type == "bytes") { - return move ? py::bytes(std::move(value)) : py::bytes(value); - } - else if (type == "none") { - return move ? py::none(std::move(value)) : py::none(value); - } - else if (type == "ellipsis") { - return move ? py::ellipsis(std::move(value)) : py::ellipsis(value); - } - else if (type == "type") { - return move ? py::type(std::move(value)) : py::type(value); - } - throw std::runtime_error("Invalid type"); - }); + return py::dict("bytes"_a = d["bytes"].cast<py::bytes>(), + "bytearray"_a = d["bytearray"].cast<py::bytearray>(), + "str"_a = d["str"].cast<py::str>(), + "bool"_a = d["bool"].cast<py::bool_>(), + "int"_a = d["int"].cast<py::int_>(), + "float"_a = d["float"].cast<py::float_>(), + "tuple"_a = d["tuple"].cast<py::tuple>(), + "list"_a = d["list"].cast<py::list>(), + "dict"_a = d["dict"].cast<py::dict>(), + "set"_a = d["set"].cast<py::set>(), + "frozenset"_a = d["frozenset"].cast<py::frozenset>(), + "memoryview"_a = d["memoryview"].cast<py::memoryview>()); + }); + + m.def("convert_to_pybind11_str", [](const py::object &o) { return py::str(o); }); + + m.def("nonconverting_constructor", + [](const std::string &type, py::object value, bool move) -> py::object { + if (type == "bytes") { + return move ? py::bytes(std::move(value)) : py::bytes(value); + } + if (type == "none") { + return move ? py::none(std::move(value)) : py::none(value); + } + if (type == "ellipsis") { + return move ? py::ellipsis(std::move(value)) : py::ellipsis(value); + } + if (type == "type") { + return move ? py::type(std::move(value)) : py::type(value); + } + throw std::runtime_error("Invalid type"); + }); m.def("get_implicit_casting", []() { py::dict d; @@ -301,10 +529,7 @@ TEST_SUBMODULE(pytypes, m) { l.append(py::cast(12)); l.append(py::int_(15)); - return py::dict( - "d"_a=d, - "l"_a=l - ); + return py::dict("d"_a = d, "l"_a = l); }); // test_print @@ -312,23 +537,27 @@ TEST_SUBMODULE(pytypes, m) { py::print("Hello, World!"); py::print(1, 2.0, "three", true, std::string("-- multiple args")); auto args = py::make_tuple("and", "a", "custom", "separator"); - py::print("*args", *args, "sep"_a="-"); - py::print("no new line here", "end"_a=" -- "); + py::print("*args", *args, "sep"_a = "-"); + py::print("no new line here", "end"_a = " -- "); py::print("next print"); auto py_stderr = py::module_::import("sys").attr("stderr"); - py::print("this goes to stderr", "file"_a=py_stderr); + py::print("this goes to stderr", "file"_a = py_stderr); - py::print("flush", "flush"_a=true); + py::print("flush", "flush"_a = true); - py::print("{a} + {b} = {c}"_s.format("a"_a="py::print", "b"_a="str.format", "c"_a="this")); + py::print( + "{a} + {b} = {c}"_s.format("a"_a = "py::print", "b"_a = "str.format", "c"_a = "this")); }); m.def("print_failure", []() { py::print(42, UnregisteredType()); }); - m.def("hash_function", [](py::object obj) { return py::hash(obj); }); + m.def("hash_function", [](py::object obj) { return py::hash(std::move(obj)); }); + + m.def("obj_contains", + [](py::object &obj, const py::object &key) { return obj.contains(key); }); - m.def("test_number_protocol", [](py::object a, py::object b) { + m.def("test_number_protocol", [](const py::object &a, const py::object &b) { py::list l; l.append(a.equal(b)); l.append(a.not_equal(b)); @@ -348,9 +577,7 @@ TEST_SUBMODULE(pytypes, m) { return l; }); - m.def("test_list_slicing", [](py::list a) { - return a[py::slice(0, -1, 2)]; - }); + m.def("test_list_slicing", [](const py::list &a) { return a[py::slice(0, -1, 2)]; }); // See #2361 m.def("issue2361_str_implicit_copy_none", []() { @@ -362,55 +589,235 @@ TEST_SUBMODULE(pytypes, m) { return is_this_none; }); - m.def("test_memoryview_object", [](py::buffer b) { - return py::memoryview(b); - }); + m.def("test_memoryview_object", [](const py::buffer &b) { return py::memoryview(b); }); - m.def("test_memoryview_buffer_info", [](py::buffer b) { - return py::memoryview(b.request()); - }); + m.def("test_memoryview_buffer_info", + [](const py::buffer &b) { return py::memoryview(b.request()); }); m.def("test_memoryview_from_buffer", [](bool is_unsigned) { - static const int16_t si16[] = { 3, 1, 4, 1, 5 }; - static const uint16_t ui16[] = { 2, 7, 1, 8 }; - if (is_unsigned) - return py::memoryview::from_buffer( - ui16, { 4 }, { sizeof(uint16_t) }); - else - return py::memoryview::from_buffer( - si16, { 5 }, { sizeof(int16_t) }); + static const int16_t si16[] = {3, 1, 4, 1, 5}; + static const uint16_t ui16[] = {2, 7, 1, 8}; + if (is_unsigned) { + return py::memoryview::from_buffer(ui16, {4}, {sizeof(uint16_t)}); + } + return py::memoryview::from_buffer(si16, {5}, {sizeof(int16_t)}); }); m.def("test_memoryview_from_buffer_nativeformat", []() { - static const char* format = "@i"; - static const int32_t arr[] = { 4, 7, 5 }; - return py::memoryview::from_buffer( - arr, sizeof(int32_t), format, { 3 }, { sizeof(int32_t) }); + static const char *format = "@i"; + static const int32_t arr[] = {4, 7, 5}; + return py::memoryview::from_buffer(arr, sizeof(int32_t), format, {3}, {sizeof(int32_t)}); }); m.def("test_memoryview_from_buffer_empty_shape", []() { - static const char* buf = ""; - return py::memoryview::from_buffer(buf, 1, "B", { }, { }); + static const char *buf = ""; + return py::memoryview::from_buffer(buf, 1, "B", {}, {}); }); m.def("test_memoryview_from_buffer_invalid_strides", []() { - static const char* buf = "\x02\x03\x04"; - return py::memoryview::from_buffer(buf, 1, "B", { 3 }, { }); + static const char *buf = "\x02\x03\x04"; + return py::memoryview::from_buffer(buf, 1, "B", {3}, {}); }); m.def("test_memoryview_from_buffer_nullptr", []() { - return py::memoryview::from_buffer( - static_cast<void*>(nullptr), 1, "B", { }, { }); + return py::memoryview::from_buffer(static_cast<void *>(nullptr), 1, "B", {}, {}); }); -#if PY_MAJOR_VERSION >= 3 m.def("test_memoryview_from_memory", []() { - const char* buf = "\xff\xe1\xab\x37"; - return py::memoryview::from_memory( - buf, static_cast<py::ssize_t>(strlen(buf))); + const char *buf = "\xff\xe1\xab\x37"; + return py::memoryview::from_memory(buf, static_cast<py::ssize_t>(strlen(buf))); }); -#endif // test_builtin_functions m.def("get_len", [](py::handle h) { return py::len(h); }); + +#ifdef PYBIND11_STR_LEGACY_PERMISSIVE + m.attr("PYBIND11_STR_LEGACY_PERMISSIVE") = true; +#endif + + m.def("isinstance_pybind11_bytes", + [](py::object o) { return py::isinstance<py::bytes>(std::move(o)); }); + m.def("isinstance_pybind11_str", + [](py::object o) { return py::isinstance<py::str>(std::move(o)); }); + + m.def("pass_to_pybind11_bytes", [](py::bytes b) { return py::len(std::move(b)); }); + m.def("pass_to_pybind11_str", [](py::str s) { return py::len(std::move(s)); }); + m.def("pass_to_std_string", [](const std::string &s) { return s.size(); }); + + // test_weakref + m.def("weakref_from_handle", [](py::handle h) { return py::weakref(h); }); + m.def("weakref_from_handle_and_function", + [](py::handle h, py::function f) { return py::weakref(h, std::move(f)); }); + m.def("weakref_from_object", [](const py::object &o) { return py::weakref(o); }); + m.def("weakref_from_object_and_function", + [](py::object o, py::function f) { return py::weakref(std::move(o), std::move(f)); }); + +// See PR #3263 for background (https://github.com/pybind/pybind11/pull/3263): +// pytypes.h could be changed to enforce the "most correct" user code below, by removing +// `const` from iterator `reference` using type aliases, but that will break existing +// user code. +#if (defined(__APPLE__) && defined(__clang__)) || defined(PYPY_VERSION) +// This is "most correct" and enforced on these platforms. +# define PYBIND11_AUTO_IT auto it +#else +// This works on many platforms and is (unfortunately) reflective of existing user code. +// NOLINTNEXTLINE(bugprone-macro-parentheses) +# define PYBIND11_AUTO_IT auto &it +#endif + + m.def("tuple_iterator", []() { + auto tup = py::make_tuple(5, 7); + int tup_sum = 0; + for (PYBIND11_AUTO_IT : tup) { + tup_sum += it.cast<int>(); + } + return tup_sum; + }); + + m.def("dict_iterator", []() { + py::dict dct; + dct[py::int_(3)] = 5; + dct[py::int_(7)] = 11; + int kv_sum = 0; + for (PYBIND11_AUTO_IT : dct) { + kv_sum += it.first.cast<int>() * 100 + it.second.cast<int>(); + } + return kv_sum; + }); + + m.def("passed_iterator", [](const py::iterator &py_it) { + int elem_sum = 0; + for (PYBIND11_AUTO_IT : py_it) { + elem_sum += it.cast<int>(); + } + return elem_sum; + }); + +#undef PYBIND11_AUTO_IT + + // Tests below this line are for pybind11 IMPLEMENTATION DETAILS: + + m.def("sequence_item_get_ssize_t", [](const py::object &o) { + return py::detail::accessor_policies::sequence_item::get(o, (py::ssize_t) 1); + }); + m.def("sequence_item_set_ssize_t", [](const py::object &o) { + auto s = py::str{"peppa", 5}; + py::detail::accessor_policies::sequence_item::set(o, (py::ssize_t) 1, s); + }); + m.def("sequence_item_get_size_t", [](const py::object &o) { + return py::detail::accessor_policies::sequence_item::get(o, (py::size_t) 2); + }); + m.def("sequence_item_set_size_t", [](const py::object &o) { + auto s = py::str{"george", 6}; + py::detail::accessor_policies::sequence_item::set(o, (py::size_t) 2, s); + }); + m.def("list_item_get_ssize_t", [](const py::object &o) { + return py::detail::accessor_policies::list_item::get(o, (py::ssize_t) 3); + }); + m.def("list_item_set_ssize_t", [](const py::object &o) { + auto s = py::str{"rebecca", 7}; + py::detail::accessor_policies::list_item::set(o, (py::ssize_t) 3, s); + }); + m.def("list_item_get_size_t", [](const py::object &o) { + return py::detail::accessor_policies::list_item::get(o, (py::size_t) 4); + }); + m.def("list_item_set_size_t", [](const py::object &o) { + auto s = py::str{"richard", 7}; + py::detail::accessor_policies::list_item::set(o, (py::size_t) 4, s); + }); + m.def("tuple_item_get_ssize_t", [](const py::object &o) { + return py::detail::accessor_policies::tuple_item::get(o, (py::ssize_t) 5); + }); + m.def("tuple_item_set_ssize_t", []() { + auto s0 = py::str{"emely", 5}; + auto s1 = py::str{"edmond", 6}; + auto o = py::tuple{2}; + py::detail::accessor_policies::tuple_item::set(o, (py::ssize_t) 0, s0); + py::detail::accessor_policies::tuple_item::set(o, (py::ssize_t) 1, s1); + return o; + }); + m.def("tuple_item_get_size_t", [](const py::object &o) { + return py::detail::accessor_policies::tuple_item::get(o, (py::size_t) 6); + }); + m.def("tuple_item_set_size_t", []() { + auto s0 = py::str{"candy", 5}; + auto s1 = py::str{"cat", 3}; + auto o = py::tuple{2}; + py::detail::accessor_policies::tuple_item::set(o, (py::size_t) 1, s1); + py::detail::accessor_policies::tuple_item::set(o, (py::size_t) 0, s0); + return o; + }); + + m.def("square_float_", [](const external::float_ &x) -> double { + double v = x.get_value(); + return v * v; + }); + + m.def("tuple_rvalue_getter", [](const py::tuple &tup) { + // tests accessing tuple object with rvalue int + for (size_t i = 0; i < tup.size(); i++) { + auto o = py::handle(tup[py::int_(i)]); + if (!o) { + throw py::value_error("tuple is malformed"); + } + } + return tup; + }); + m.def("list_rvalue_getter", [](const py::list &l) { + // tests accessing list with rvalue int + for (size_t i = 0; i < l.size(); i++) { + auto o = py::handle(l[py::int_(i)]); + if (!o) { + throw py::value_error("list is malformed"); + } + } + return l; + }); + m.def("populate_dict_rvalue", [](int population) { + auto d = py::dict(); + for (int i = 0; i < population; i++) { + d[py::int_(i)] = py::int_(i); + } + return d; + }); + m.def("populate_obj_str_attrs", [](py::object &o, int population) { + for (int i = 0; i < population; i++) { + o.attr(py::str(py::int_(i))) = py::str(py::int_(i)); + } + return o; + }); + + // testing immutable object augmented assignment: #issue 3812 + m.def("inplace_append", [](py::object &a, const py::object &b) { + a += b; + return a; + }); + m.def("inplace_subtract", [](py::object &a, const py::object &b) { + a -= b; + return a; + }); + m.def("inplace_multiply", [](py::object &a, const py::object &b) { + a *= b; + return a; + }); + m.def("inplace_divide", [](py::object &a, const py::object &b) { + a /= b; + return a; + }); + m.def("inplace_or", [](py::object &a, const py::object &b) { + a |= b; + return a; + }); + m.def("inplace_and", [](py::object &a, const py::object &b) { + a &= b; + return a; + }); + m.def("inplace_lshift", [](py::object &a, const py::object &b) { + a <<= b; + return a; + }); + m.def("inplace_rshift", [](py::object &a, const py::object &b) { + a >>= b; + return a; + }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index b1509a02..eda7a20a 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1,12 +1,27 @@ -# -*- coding: utf-8 -*- -from __future__ import division -import pytest +import contextlib import sys +import types -import env # noqa: F401 +import pytest +import env +from pybind11_tests import detailed_error_messages_enabled from pybind11_tests import pytypes as m -from pybind11_tests import debug_enabled + + +def test_obj_class_name(): + assert m.obj_class_name(None) == "NoneType" + assert m.obj_class_name(list) == "list" + assert m.obj_class_name([]) == "list" + + +def test_handle_from_move_only_type_with_operator_PyObject(): + assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() + assert m.handle_from_move_only_type_with_operator_PyObject_const() + + +def test_bool(doc): + assert doc(m.get_bool) == "get_bool() -> bool" def test_int(doc): @@ -17,11 +32,40 @@ def test_iterator(doc): assert doc(m.get_iterator) == "get_iterator() -> Iterator" +@pytest.mark.parametrize( + ("pytype", "from_iter_func"), + [ + (frozenset, m.get_frozenset_from_iterable), + (list, m.get_list_from_iterable), + (set, m.get_set_from_iterable), + (tuple, m.get_tuple_from_iterable), + ], +) +def test_from_iterable(pytype, from_iter_func): + my_iter = iter(range(10)) + s = from_iter_func(my_iter) + assert type(s) == pytype + assert s == pytype(range(10)) + + def test_iterable(doc): assert doc(m.get_iterable) == "get_iterable() -> Iterable" +def test_float(doc): + assert doc(m.get_float) == "get_float() -> float" + + def test_list(capture, doc): + assert m.list_no_args() == [] + assert m.list_ssize_t() == [] + assert m.list_size_t() == [] + lins = [1, 2] + m.list_insert_ssize_t(lins) + assert lins == [1, 83, 2] + m.list_insert_size_t(lins) + assert lins == [1, 83, 2, 57] + with capture: lst = m.get_list() assert lst == ["inserted-0", "overwritten", "inserted-2"] @@ -43,18 +87,19 @@ def test_list(capture, doc): assert doc(m.print_list) == "print_list(arg0: list) -> None" -def test_none(capture, doc): +def test_none(doc): assert doc(m.get_none) == "get_none() -> None" assert doc(m.print_none) == "print_none(arg0: None) -> None" def test_set(capture, doc): s = m.get_set() + assert isinstance(s, set) assert s == {"key1", "key2", "key3"} + s.add("key4") with capture: - s.add("key4") - m.print_set(s) + m.print_anyset(s) assert ( capture.unordered == """ @@ -65,12 +110,43 @@ def test_set(capture, doc): """ ) - assert not m.set_contains(set([]), 42) - assert m.set_contains({42}, 42) - assert m.set_contains({"foo"}, "foo") + m.set_add(s, "key5") + assert m.anyset_size(s) == 5 - assert doc(m.get_list) == "get_list() -> list" - assert doc(m.print_list) == "print_list(arg0: list) -> None" + m.set_clear(s) + assert m.anyset_empty(s) + + assert not m.anyset_contains(set(), 42) + assert m.anyset_contains({42}, 42) + assert m.anyset_contains({"foo"}, "foo") + + assert doc(m.get_set) == "get_set() -> set" + assert doc(m.print_anyset) == "print_anyset(arg0: anyset) -> None" + + +def test_frozenset(capture, doc): + s = m.get_frozenset() + assert isinstance(s, frozenset) + assert s == frozenset({"key1", "key2", "key3"}) + + with capture: + m.print_anyset(s) + assert ( + capture.unordered + == """ + key: key1 + key: key2 + key: key3 + """ + ) + assert m.anyset_size(s) == 3 + assert not m.anyset_empty(s) + + assert not m.anyset_contains(frozenset(), 42) + assert m.anyset_contains(frozenset({42}), 42) + assert m.anyset_contains(frozenset({"foo"}), "foo") + + assert doc(m.get_frozenset) == "get_frozenset() -> frozenset" def test_dict(capture, doc): @@ -98,13 +174,55 @@ def test_dict(capture, doc): assert m.dict_keyword_constructor() == {"x": 1, "y": 2, "z": 3} +class CustomContains: + d = {"key": None} + + def __contains__(self, m): + return m in self.d + + +@pytest.mark.parametrize( + ("arg", "func"), + [ + (set(), m.anyset_contains), + ({}, m.dict_contains), + (CustomContains(), m.obj_contains), + ], +) +@pytest.mark.xfail("env.PYPY and sys.pypy_version_info < (7, 3, 10)", strict=False) +def test_unhashable_exceptions(arg, func): + class Unhashable: + __hash__ = None + + with pytest.raises(TypeError) as exc_info: + func(arg, Unhashable()) + assert "unhashable type:" in str(exc_info.value) + + +def test_tuple(): + assert m.tuple_no_args() == () + assert m.tuple_ssize_t() == () + assert m.tuple_size_t() == () + assert m.get_tuple() == (42, None, "spam") + + +def test_simple_namespace(): + ns = m.get_simple_namespace() + assert ns.attr == 42 + assert ns.x == "foo" + assert ns.right == 2 + assert not hasattr(ns, "wrong") + + def test_str(doc): + assert m.str_from_char_ssize_t().encode().decode() == "red" + assert m.str_from_char_size_t().encode().decode() == "blue" assert m.str_from_string().encode().decode() == "baz" assert m.str_from_bytes().encode().decode() == "boo" assert doc(m.str_from_bytes) == "str_from_bytes() -> str" - class A(object): + class A: def __str__(self): return "this is a str" @@ -120,24 +238,46 @@ def test_str(doc): assert s1 == s2 malformed_utf8 = b"\x80" - assert m.str_from_object(malformed_utf8) is malformed_utf8 # To be fixed; see #2380 - if env.PY2: - # with pytest.raises(UnicodeDecodeError): - # m.str_from_object(malformed_utf8) - with pytest.raises(UnicodeDecodeError): - m.str_from_handle(malformed_utf8) + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert m.str_from_object(malformed_utf8) is malformed_utf8 else: - # assert m.str_from_object(malformed_utf8) == "b'\\x80'" - assert m.str_from_handle(malformed_utf8) == "b'\\x80'" + assert m.str_from_object(malformed_utf8) == "b'\\x80'" + assert m.str_from_handle(malformed_utf8) == "b'\\x80'" + + assert m.str_from_string_from_str("this is a str") == "this is a str" + ucs_surrogates_str = "\udcc3" + with pytest.raises(UnicodeEncodeError): + m.str_from_string_from_str(ucs_surrogates_str) + + +@pytest.mark.parametrize( + "func", + [ + m.str_from_bytes_input, + m.str_from_cstr_input, + m.str_from_std_string_input, + ], +) +def test_surrogate_pairs_unicode_error(func): + input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") + with pytest.raises(UnicodeDecodeError): + func(input_str) def test_bytes(doc): + assert m.bytes_from_char_ssize_t().decode() == "green" + assert m.bytes_from_char_size_t().decode() == "purple" assert m.bytes_from_string().decode() == "foo" assert m.bytes_from_str().decode() == "bar" - assert doc(m.bytes_from_str) == "bytes_from_str() -> {}".format( - "str" if env.PY2 else "bytes" - ) + assert doc(m.bytes_from_str) == "bytes_from_str() -> bytes" + + +def test_bytearray(): + assert m.bytearray_from_char_ssize_t().decode() == "$%" + assert m.bytearray_from_char_size_t().decode() == "@$!" + assert m.bytearray_from_string().decode() == "foo" + assert m.bytearray_size() == len("foo") def test_capsule(capture): @@ -155,6 +295,19 @@ def test_capsule(capture): ) with capture: + a = m.return_renamed_capsule_with_destructor() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + renaming capsule + destructing capsule + """ + ) + + with capture: a = m.return_capsule_with_destructor_2() del a pytest.gc_collect() @@ -167,6 +320,32 @@ def test_capsule(capture): ) with capture: + a = m.return_capsule_with_destructor_3() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + destructing capsule: 1233 + original name: oname + """ + ) + + with capture: + a = m.return_renamed_capsule_with_destructor_2() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + renaming capsule + destructing capsule: 1234 + """ + ) + + with capture: a = m.return_capsule_with_name_and_destructor() del a pytest.gc_collect() @@ -178,6 +357,17 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_explicit_nullptr_dtor() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule with explicit nullptr dtor + """ + ) + def test_accessors(): class SubTestObject: @@ -208,7 +398,7 @@ def test_accessors(): assert d["implicit_list"] == [1, 2, 3] assert all(x in TestObject.__dict__ for x in d["implicit_dict"]) - assert m.tuple_accessor(tuple()) == (0, 1, 2) + assert m.tuple_accessor(()) == (0, 1, 2) d = m.accessor_assignment() assert d["get"] == 0 @@ -218,19 +408,23 @@ def test_accessors(): assert d["var"] == 99 +def test_accessor_moves(): + inc_refs = m.accessor_moves() + if inc_refs: + assert inc_refs == [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0] + else: + pytest.skip("Not defined: PYBIND11_HANDLE_REF_DEBUG") + + def test_constructors(): """C++ default and converting constructors are equivalent to type calls in Python""" - types = [bytes, str, bool, int, float, tuple, list, dict, set] + types = [bytes, bytearray, str, bool, int, float, tuple, list, dict, set] expected = {t.__name__: t() for t in types} - if env.PY2: - # Note that bytes.__name__ == 'str' in Python 2. - # pybind11::str is unicode even under Python 2. - expected["bytes"] = bytes() - expected["str"] = unicode() # noqa: F821 assert m.default_constructors() == expected data = { bytes: b"41", # Currently no supported or working conversions. + bytearray: bytearray(b"41"), str: 42, bool: "Not empty", int: "42", @@ -239,15 +433,11 @@ def test_constructors(): list: range(3), dict: [("two", 2), ("one", 1), ("three", 3)], set: [4, 4, 5, 6, 6, 6], + frozenset: [4, 4, 5, 6, 6, 6], memoryview: b"abc", } inputs = {k.__name__: v for k, v in data.items()} expected = {k.__name__: k(v) for k, v in data.items()} - if env.PY2: # Similar to the above. See comments above. - inputs["bytes"] = b"41" - inputs["str"] = 42 - expected["bytes"] = b"41" - expected["str"] = u"42" assert m.converting_constructors(inputs) == expected assert m.cast_functions(inputs) == expected @@ -274,8 +464,8 @@ def test_non_converting_constructors(): for move in [True, False]: with pytest.raises(TypeError) as excinfo: m.nonconverting_constructor(t, v, move) - expected_error = "Object of type '{}' is not an instance of '{}'".format( - type(v).__name__, t + expected_error = ( + f"Object of type '{type(v).__name__}' is not an instance of '{t}'" ) assert str(excinfo.value) == expected_error @@ -283,33 +473,39 @@ def test_non_converting_constructors(): def test_pybind11_str_raw_str(): # specifically to exercise pybind11::str::raw_str cvt = m.convert_to_pybind11_str - assert cvt(u"Str") == u"Str" - assert cvt(b"Bytes") == u"Bytes" if env.PY2 else "b'Bytes'" - assert cvt(None) == u"None" - assert cvt(False) == u"False" - assert cvt(True) == u"True" - assert cvt(42) == u"42" - assert cvt(2 ** 65) == u"36893488147419103232" - assert cvt(-1.50) == u"-1.5" - assert cvt(()) == u"()" - assert cvt((18,)) == u"(18,)" - assert cvt([]) == u"[]" - assert cvt([28]) == u"[28]" - assert cvt({}) == u"{}" - assert cvt({3: 4}) == u"{3: 4}" - assert cvt(set()) == u"set([])" if env.PY2 else "set()" - assert cvt({3, 3}) == u"set([3])" if env.PY2 else "{3}" - - valid_orig = u"DZ" + assert cvt("Str") == "Str" + assert cvt(b"Bytes") == "b'Bytes'" + assert cvt(None) == "None" + assert cvt(False) == "False" + assert cvt(True) == "True" + assert cvt(42) == "42" + assert cvt(2**65) == "36893488147419103232" + assert cvt(-1.50) == "-1.5" + assert cvt(()) == "()" + assert cvt((18,)) == "(18,)" + assert cvt([]) == "[]" + assert cvt([28]) == "[28]" + assert cvt({}) == "{}" + assert cvt({3: 4}) == "{3: 4}" + assert cvt(set()) == "set()" + assert cvt({3}) == "{3}" + + valid_orig = "DZ" valid_utf8 = valid_orig.encode("utf-8") valid_cvt = cvt(valid_utf8) - assert type(valid_cvt) == bytes # Probably surprising. - assert valid_cvt == b"\xc7\xb1" + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert valid_cvt is valid_utf8 + else: + assert type(valid_cvt) is str + assert valid_cvt == "b'\\xc7\\xb1'" malformed_utf8 = b"\x80" - malformed_cvt = cvt(malformed_utf8) - assert type(malformed_cvt) == bytes # Probably surprising. - assert malformed_cvt == b"\x80" + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert cvt(malformed_utf8) is malformed_utf8 + else: + malformed_cvt = cvt(malformed_utf8) + assert type(malformed_cvt) is str + assert malformed_cvt == "b'\\x80'" def test_implicit_casting(): @@ -350,22 +546,22 @@ def test_print(capture): with pytest.raises(RuntimeError) as excinfo: m.print_failure() - assert str(excinfo.value) == "make_tuple(): unable to convert " + ( - "argument of type 'UnregisteredType' to Python object" - if debug_enabled - else "arguments to Python object (compile in debug mode for details)" + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'1' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" ) def test_hash(): - class Hashable(object): + class Hashable: def __init__(self, value): self.value = value def __hash__(self): return self.value - class Unhashable(object): + class Unhashable: __hash__ = None assert m.hash_function(Hashable(42)) == 42 @@ -405,11 +601,12 @@ def test_issue2361(): assert m.issue2361_str_implicit_copy_none() == "None" with pytest.raises(TypeError) as excinfo: assert m.issue2361_dict_implicit_copy_none() - assert "'NoneType' object is not iterable" in str(excinfo.value) + assert "NoneType" in str(excinfo.value) + assert "iterable" in str(excinfo.value) @pytest.mark.parametrize( - "method, args, fmt, expected_view", + ("method", "args", "fmt", "expected_view"), [ (m.test_memoryview_object, (b"red",), "B", b"red"), (m.test_memoryview_buffer_info, (b"green",), "B", b"green"), @@ -422,12 +619,7 @@ def test_memoryview(method, args, fmt, expected_view): view = method(*args) assert isinstance(view, memoryview) assert view.format == fmt - if isinstance(expected_view, bytes) or not env.PY2: - view_as_list = list(view) - else: - # Using max to pick non-zero byte (big-endian vs little-endian). - view_as_list = [max([ord(c) for c in s]) for s in view] - assert view_as_list == list(expected_view) + assert list(view) == list(expected_view) @pytest.mark.xfail("env.PYPY", reason="getrefcount is not available") @@ -451,12 +643,7 @@ def test_memoryview_from_buffer_empty_shape(): view = m.test_memoryview_from_buffer_empty_shape() assert isinstance(view, memoryview) assert view.format == "B" - if env.PY2: - # Python 2 behavior is weird, but Python 3 (the future) is fine. - # PyPy3 has <memoryview, while CPython 2 has <memory - assert bytes(view).startswith(b"<memory") - else: - assert bytes(view) == b"" + assert bytes(view) == b"" def test_test_memoryview_from_buffer_invalid_strides(): @@ -465,14 +652,10 @@ def test_test_memoryview_from_buffer_invalid_strides(): def test_test_memoryview_from_buffer_nullptr(): - if env.PY2: + with pytest.raises(ValueError): m.test_memoryview_from_buffer_nullptr() - else: - with pytest.raises(ValueError): - m.test_memoryview_from_buffer_nullptr() -@pytest.mark.skipif("env.PY2") def test_memoryview_from_memory(): view = m.test_memoryview_from_memory() assert isinstance(view, memoryview) @@ -481,10 +664,235 @@ def test_memoryview_from_memory(): def test_builtin_functions(): - assert m.get_len([i for i in range(42)]) == 42 + assert m.get_len(list(range(42))) == 42 with pytest.raises(TypeError) as exc_info: m.get_len(i for i in range(42)) assert str(exc_info.value) in [ "object of type 'generator' has no len()", "'generator' has no length", ] # PyPy + + +def test_isinstance_string_types(): + assert m.isinstance_pybind11_bytes(b"") + assert not m.isinstance_pybind11_bytes("") + + assert m.isinstance_pybind11_str("") + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert m.isinstance_pybind11_str(b"") + else: + assert not m.isinstance_pybind11_str(b"") + + +def test_pass_bytes_or_unicode_to_string_types(): + assert m.pass_to_pybind11_bytes(b"Bytes") == 5 + with pytest.raises(TypeError): + m.pass_to_pybind11_bytes("Str") + + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert m.pass_to_pybind11_str(b"Bytes") == 5 + else: + with pytest.raises(TypeError): + m.pass_to_pybind11_str(b"Bytes") + assert m.pass_to_pybind11_str("Str") == 3 + + assert m.pass_to_std_string(b"Bytes") == 5 + assert m.pass_to_std_string("Str") == 3 + + malformed_utf8 = b"\x80" + if hasattr(m, "PYBIND11_STR_LEGACY_PERMISSIVE"): + assert m.pass_to_pybind11_str(malformed_utf8) == 1 + else: + with pytest.raises(TypeError): + m.pass_to_pybind11_str(malformed_utf8) + + +@pytest.mark.parametrize( + ("create_weakref", "create_weakref_with_callback"), + [ + (m.weakref_from_handle, m.weakref_from_handle_and_function), + (m.weakref_from_object, m.weakref_from_object_and_function), + ], +) +def test_weakref(create_weakref, create_weakref_with_callback): + from weakref import getweakrefcount + + # Apparently, you cannot weakly reference an object() + class WeaklyReferenced: + pass + + callback_called = False + + def callback(_): + nonlocal callback_called + callback_called = True + + obj = WeaklyReferenced() + assert getweakrefcount(obj) == 0 + wr = create_weakref(obj) + assert getweakrefcount(obj) == 1 + + obj = WeaklyReferenced() + assert getweakrefcount(obj) == 0 + wr = create_weakref_with_callback(obj, callback) # noqa: F841 + assert getweakrefcount(obj) == 1 + assert not callback_called + del obj + pytest.gc_collect() + assert callback_called + + +@pytest.mark.parametrize( + ("create_weakref", "has_callback"), + [ + (m.weakref_from_handle, False), + (m.weakref_from_object, False), + (m.weakref_from_handle_and_function, True), + (m.weakref_from_object_and_function, True), + ], +) +def test_weakref_err(create_weakref, has_callback): + class C: + __slots__ = [] + + def callback(_): + pass + + ob = C() + # Should raise TypeError on CPython + with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext(): + _ = create_weakref(ob, callback) if has_callback else create_weakref(ob) + + +def test_cpp_iterators(): + assert m.tuple_iterator() == 12 + assert m.dict_iterator() == 305 + 711 + assert m.passed_iterator(iter((-7, 3))) == -4 + + +def test_implementation_details(): + lst = [39, 43, 92, 49, 22, 29, 93, 98, 26, 57, 8] + tup = tuple(lst) + assert m.sequence_item_get_ssize_t(lst) == 43 + assert m.sequence_item_set_ssize_t(lst) is None + assert lst[1] == "peppa" + assert m.sequence_item_get_size_t(lst) == 92 + assert m.sequence_item_set_size_t(lst) is None + assert lst[2] == "george" + assert m.list_item_get_ssize_t(lst) == 49 + assert m.list_item_set_ssize_t(lst) is None + assert lst[3] == "rebecca" + assert m.list_item_get_size_t(lst) == 22 + assert m.list_item_set_size_t(lst) is None + assert lst[4] == "richard" + assert m.tuple_item_get_ssize_t(tup) == 29 + assert m.tuple_item_set_ssize_t() == ("emely", "edmond") + assert m.tuple_item_get_size_t(tup) == 93 + assert m.tuple_item_set_size_t() == ("candy", "cat") + + +def test_external_float_(): + r1 = m.square_float_(2.0) + assert r1 == 4.0 + + +def test_tuple_rvalue_getter(): + pop = 1000 + tup = tuple(range(pop)) + m.tuple_rvalue_getter(tup) + + +def test_list_rvalue_getter(): + pop = 1000 + my_list = list(range(pop)) + m.list_rvalue_getter(my_list) + + +def test_populate_dict_rvalue(): + pop = 1000 + my_dict = {i: i for i in range(pop)} + assert m.populate_dict_rvalue(pop) == my_dict + + +def test_populate_obj_str_attrs(): + pop = 1000 + o = types.SimpleNamespace(**{str(i): i for i in range(pop)}) + new_o = m.populate_obj_str_attrs(o, pop) + new_attrs = {k: v for k, v in new_o.__dict__.items() if not k.startswith("_")} + assert all(isinstance(v, str) for v in new_attrs.values()) + assert len(new_attrs) == pop + + +@pytest.mark.parametrize( + ("a", "b"), + [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))], +) +def test_inplace_append(a, b): + expected = a + b + assert m.inplace_append(a, b) == expected + + +@pytest.mark.parametrize( + ("a", "b"), [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))] +) +def test_inplace_subtract(a, b): + expected = a - b + assert m.inplace_subtract(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(3, 2), (3.0, 2.0), ([1], 3)]) +def test_inplace_multiply(a, b): + expected = a * b + assert m.inplace_multiply(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(6, 3), (6.0, 3.0)]) +def test_inplace_divide(a, b): + expected = a / b + assert m.inplace_divide(a, b) == expected + + +@pytest.mark.parametrize( + ("a", "b"), + [ + (False, True), + ( + set(), + { + 1, + }, + ), + ], +) +def test_inplace_or(a, b): + expected = a | b + assert m.inplace_or(a, b) == expected + + +@pytest.mark.parametrize( + ("a", "b"), + [ + (True, False), + ( + {1, 2, 3}, + { + 1, + }, + ), + ], +) +def test_inplace_and(a, b): + expected = a & b + assert m.inplace_and(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-3, 2)]) +def test_inplace_lshift(a, b): + expected = a << b + assert m.inplace_lshift(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-2, 2)]) +def test_inplace_rshift(a, b): + expected = a >> b + assert m.inplace_rshift(a, b) == expected diff --git a/tests/test_sequences_and_iterators.cpp b/tests/test_sequences_and_iterators.cpp index d318052a..1de65edb 100644 --- a/tests/test_sequences_and_iterators.cpp +++ b/tests/test_sequences_and_iterators.cpp @@ -8,38 +8,91 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/operators.h> #include <pybind11/stl.h> +#include "constructor_stats.h" +#include "pybind11_tests.h" + #include <algorithm> +#include <utility> +#include <vector> + +#ifdef PYBIND11_HAS_OPTIONAL +# include <optional> +#endif // PYBIND11_HAS_OPTIONAL -template<typename T> +template <typename T> class NonZeroIterator { - const T* ptr_; + const T *ptr_; + public: - NonZeroIterator(const T* ptr) : ptr_(ptr) {} - const T& operator*() const { return *ptr_; } - NonZeroIterator& operator++() { ++ptr_; return *this; } + explicit NonZeroIterator(const T *ptr) : ptr_(ptr) {} + const T &operator*() const { return *ptr_; } + NonZeroIterator &operator++() { + ++ptr_; + return *this; + } }; class NonZeroSentinel {}; -template<typename A, typename B> -bool operator==(const NonZeroIterator<std::pair<A, B>>& it, const NonZeroSentinel&) { +template <typename A, typename B> +bool operator==(const NonZeroIterator<std::pair<A, B>> &it, const NonZeroSentinel &) { return !(*it).first || !(*it).second; } +/* Iterator where dereferencing returns prvalues instead of references. */ +template <typename T> +class NonRefIterator { + const T *ptr_; + +public: + explicit NonRefIterator(const T *ptr) : ptr_(ptr) {} + T operator*() const { return T(*ptr_); } + NonRefIterator &operator++() { + ++ptr_; + return *this; + } + bool operator==(const NonRefIterator &other) const { return ptr_ == other.ptr_; } +}; + +class NonCopyableInt { +public: + explicit NonCopyableInt(int value) : value_(value) {} + NonCopyableInt(const NonCopyableInt &) = delete; + NonCopyableInt(NonCopyableInt &&other) noexcept : value_(other.value_) { + other.value_ = -1; // detect when an unwanted move occurs + } + NonCopyableInt &operator=(const NonCopyableInt &) = delete; + NonCopyableInt &operator=(NonCopyableInt &&other) noexcept { + value_ = other.value_; + other.value_ = -1; // detect when an unwanted move occurs + return *this; + } + int get() const { return value_; } + void set(int value) { value_ = value; } + ~NonCopyableInt() = default; + +private: + int value_; +}; +using NonCopyableIntPair = std::pair<NonCopyableInt, NonCopyableInt>; +PYBIND11_MAKE_OPAQUE(std::vector<NonCopyableInt>); +PYBIND11_MAKE_OPAQUE(std::vector<NonCopyableIntPair>); + template <typename PythonType> py::list test_random_access_iterator(PythonType x) { - if (x.size() < 5) + if (x.size() < 5) { throw py::value_error("Please provide at least 5 elements for testing."); + } auto checks = py::list(); auto assert_equal = [&checks](py::handle a, py::handle b) { auto result = PyObject_RichCompareBool(a.ptr(), b.ptr(), Py_EQ); - if (result == -1) { throw py::error_already_set(); } + if (result == -1) { + throw py::error_already_set(); + } checks.append(result != 0); }; @@ -74,63 +127,83 @@ py::list test_random_access_iterator(PythonType x) { TEST_SUBMODULE(sequences_and_iterators, m) { // test_sliceable - class Sliceable{ + class Sliceable { public: - Sliceable(int n): size(n) {} - int start,stop,step; - int size; + explicit Sliceable(int n) : size(n) {} + int start, stop, step; + int size; }; - py::class_<Sliceable>(m,"Sliceable") + py::class_<Sliceable>(m, "Sliceable") .def(py::init<int>()) - .def("__getitem__",[](const Sliceable &s, py::slice slice) { - py::ssize_t start, stop, step, slicelength; - if (!slice.compute(s.size, &start, &stop, &step, &slicelength)) - throw py::error_already_set(); - int istart = static_cast<int>(start); - int istop = static_cast<int>(stop); - int istep = static_cast<int>(step); - return std::make_tuple(istart,istop,istep); - }) - ; + .def("__getitem__", [](const Sliceable &s, const py::slice &slice) { + py::ssize_t start = 0, stop = 0, step = 0, slicelength = 0; + if (!slice.compute(s.size, &start, &stop, &step, &slicelength)) { + throw py::error_already_set(); + } + int istart = static_cast<int>(start); + int istop = static_cast<int>(stop); + int istep = static_cast<int>(step); + return std::make_tuple(istart, istop, istep); + }); + + m.def("make_forward_slice_size_t", []() { return py::slice(0, -1, 1); }); + m.def("make_reversed_slice_object", + []() { return py::slice(py::none(), py::none(), py::int_(-1)); }); +#ifdef PYBIND11_HAS_OPTIONAL + m.attr("has_optional") = true; + m.def("make_reversed_slice_size_t_optional_verbose", + []() { return py::slice(std::nullopt, std::nullopt, -1); }); + // Warning: The following spelling may still compile if optional<> is not present and give + // wrong answers. Please use with caution. + m.def("make_reversed_slice_size_t_optional", []() { return py::slice({}, {}, -1); }); +#else + m.attr("has_optional") = false; +#endif // test_sequence class Sequence { public: - Sequence(size_t size) : m_size(size) { + explicit Sequence(size_t size) : m_size(size) { print_created(this, "of size", m_size); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) m_data = new float[size]; memset(m_data, 0, sizeof(float) * size); } - Sequence(const std::vector<float> &value) : m_size(value.size()) { + explicit Sequence(const std::vector<float> &value) : m_size(value.size()) { print_created(this, "of size", m_size, "from std::vector"); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) m_data = new float[m_size]; memcpy(m_data, &value[0], sizeof(float) * m_size); } Sequence(const Sequence &s) : m_size(s.m_size) { print_copy_created(this); + // NOLINTNEXTLINE(cppcoreguidelines-prefer-member-initializer) m_data = new float[m_size]; - memcpy(m_data, s.m_data, sizeof(float)*m_size); + memcpy(m_data, s.m_data, sizeof(float) * m_size); } - Sequence(Sequence &&s) : m_size(s.m_size), m_data(s.m_data) { + Sequence(Sequence &&s) noexcept : m_size(s.m_size), m_data(s.m_data) { print_move_created(this); s.m_size = 0; s.m_data = nullptr; } - ~Sequence() { print_destroyed(this); delete[] m_data; } + ~Sequence() { + print_destroyed(this); + delete[] m_data; + } Sequence &operator=(const Sequence &s) { if (&s != this) { delete[] m_data; m_size = s.m_size; m_data = new float[m_size]; - memcpy(m_data, s.m_data, sizeof(float)*m_size); + memcpy(m_data, s.m_data, sizeof(float) * m_size); } print_copy_assigned(this); return *this; } - Sequence &operator=(Sequence &&s) { + Sequence &operator=(Sequence &&s) noexcept { if (&s != this) { delete[] m_data; m_size = s.m_size; @@ -143,10 +216,14 @@ TEST_SUBMODULE(sequences_and_iterators, m) { } bool operator==(const Sequence &s) const { - if (m_size != s.size()) return false; - for (size_t i = 0; i < m_size; ++i) - if (m_data[i] != s[i]) + if (m_size != s.size()) { + return false; + } + for (size_t i = 0; i < m_size; ++i) { + if (m_data[i] != s[i]) { return false; + } + } return true; } bool operator!=(const Sequence &s) const { return !operator==(s); } @@ -155,23 +232,26 @@ TEST_SUBMODULE(sequences_and_iterators, m) { float &operator[](size_t index) { return m_data[index]; } bool contains(float v) const { - for (size_t i = 0; i < m_size; ++i) - if (v == m_data[i]) + for (size_t i = 0; i < m_size; ++i) { + if (v == m_data[i]) { return true; + } + } return false; } Sequence reversed() const { Sequence result(m_size); - for (size_t i = 0; i < m_size; ++i) + for (size_t i = 0; i < m_size; ++i) { result[m_size - i - 1] = m_data[i]; + } return result; } size_t size() const { return m_size; } const float *begin() const { return m_data; } - const float *end() const { return m_data+m_size; } + const float *end() const { return m_data + m_size; } private: size_t m_size; @@ -179,43 +259,59 @@ TEST_SUBMODULE(sequences_and_iterators, m) { }; py::class_<Sequence>(m, "Sequence") .def(py::init<size_t>()) - .def(py::init<const std::vector<float>&>()) + .def(py::init<const std::vector<float> &>()) /// Bare bones interface - .def("__getitem__", [](const Sequence &s, size_t i) { - if (i >= s.size()) throw py::index_error(); - return s[i]; - }) - .def("__setitem__", [](Sequence &s, size_t i, float v) { - if (i >= s.size()) throw py::index_error(); - s[i] = v; - }) + .def("__getitem__", + [](const Sequence &s, size_t i) { + if (i >= s.size()) { + throw py::index_error(); + } + return s[i]; + }) + .def("__setitem__", + [](Sequence &s, size_t i, float v) { + if (i >= s.size()) { + throw py::index_error(); + } + s[i] = v; + }) .def("__len__", &Sequence::size) /// Optional sequence protocol operations - .def("__iter__", [](const Sequence &s) { return py::make_iterator(s.begin(), s.end()); }, - py::keep_alive<0, 1>() /* Essential: keep object alive while iterator exists */) + .def( + "__iter__", + [](const Sequence &s) { return py::make_iterator(s.begin(), s.end()); }, + py::keep_alive<0, 1>() /* Essential: keep object alive while iterator exists */) .def("__contains__", [](const Sequence &s, float v) { return s.contains(v); }) .def("__reversed__", [](const Sequence &s) -> Sequence { return s.reversed(); }) /// Slicing protocol (optional) - .def("__getitem__", [](const Sequence &s, py::slice slice) -> Sequence* { - size_t start, stop, step, slicelength; - if (!slice.compute(s.size(), &start, &stop, &step, &slicelength)) - throw py::error_already_set(); - auto *seq = new Sequence(slicelength); - for (size_t i = 0; i < slicelength; ++i) { - (*seq)[i] = s[start]; start += step; - } - return seq; - }) - .def("__setitem__", [](Sequence &s, py::slice slice, const Sequence &value) { - size_t start, stop, step, slicelength; - if (!slice.compute(s.size(), &start, &stop, &step, &slicelength)) - throw py::error_already_set(); - if (slicelength != value.size()) - throw std::runtime_error("Left and right hand size of slice assignment have different sizes!"); - for (size_t i = 0; i < slicelength; ++i) { - s[start] = value[i]; start += step; - } - }) + .def("__getitem__", + [](const Sequence &s, const py::slice &slice) -> Sequence * { + size_t start = 0, stop = 0, step = 0, slicelength = 0; + if (!slice.compute(s.size(), &start, &stop, &step, &slicelength)) { + throw py::error_already_set(); + } + auto *seq = new Sequence(slicelength); + for (size_t i = 0; i < slicelength; ++i) { + (*seq)[i] = s[start]; + start += step; + } + return seq; + }) + .def("__setitem__", + [](Sequence &s, const py::slice &slice, const Sequence &value) { + size_t start = 0, stop = 0, step = 0, slicelength = 0; + if (!slice.compute(s.size(), &start, &stop, &step, &slicelength)) { + throw py::error_already_set(); + } + if (slicelength != value.size()) { + throw std::runtime_error( + "Left and right hand size of slice assignment have different sizes!"); + } + for (size_t i = 0; i < slicelength; ++i) { + s[start] = value[i]; + start += step; + } + }) /// Comparisons .def(py::self == py::self) .def(py::self != py::self) @@ -223,19 +319,21 @@ TEST_SUBMODULE(sequences_and_iterators, m) { ; // test_map_iterator - // Interface of a map-like object that isn't (directly) an unordered_map, but provides some basic - // map-like functionality. + // Interface of a map-like object that isn't (directly) an unordered_map, but provides some + // basic map-like functionality. class StringMap { public: StringMap() = default; - StringMap(std::unordered_map<std::string, std::string> init) + explicit StringMap(std::unordered_map<std::string, std::string> init) : map(std::move(init)) {} - void set(std::string key, std::string val) { map[key] = val; } - std::string get(std::string key) const { return map.at(key); } + void set(const std::string &key, std::string val) { map[key] = std::move(val); } + std::string get(const std::string &key) const { return map.at(key); } size_t size() const { return map.size(); } + private: std::unordered_map<std::string, std::string> map; + public: decltype(map.cbegin()) begin() const { return map.cbegin(); } decltype(map.cend()) end() const { return map.cend(); } @@ -243,38 +341,142 @@ TEST_SUBMODULE(sequences_and_iterators, m) { py::class_<StringMap>(m, "StringMap") .def(py::init<>()) .def(py::init<std::unordered_map<std::string, std::string>>()) - .def("__getitem__", [](const StringMap &map, std::string key) { - try { return map.get(key); } - catch (const std::out_of_range&) { - throw py::key_error("key '" + key + "' does not exist"); - } - }) + .def("__getitem__", + [](const StringMap &map, const std::string &key) { + try { + return map.get(key); + } catch (const std::out_of_range &) { + throw py::key_error("key '" + key + "' does not exist"); + } + }) .def("__setitem__", &StringMap::set) .def("__len__", &StringMap::size) - .def("__iter__", [](const StringMap &map) { return py::make_key_iterator(map.begin(), map.end()); }, - py::keep_alive<0, 1>()) - .def("items", [](const StringMap &map) { return py::make_iterator(map.begin(), map.end()); }, - py::keep_alive<0, 1>()) - ; + .def( + "__iter__", + [](const StringMap &map) { return py::make_key_iterator(map.begin(), map.end()); }, + py::keep_alive<0, 1>()) + .def( + "items", + [](const StringMap &map) { return py::make_iterator(map.begin(), map.end()); }, + py::keep_alive<0, 1>()) + .def( + "values", + [](const StringMap &map) { return py::make_value_iterator(map.begin(), map.end()); }, + py::keep_alive<0, 1>()); // test_generalized_iterators class IntPairs { public: - IntPairs(std::vector<std::pair<int, int>> data) : data_(std::move(data)) {} - const std::pair<int, int>* begin() const { return data_.data(); } + explicit IntPairs(std::vector<std::pair<int, int>> data) : data_(std::move(data)) {} + const std::pair<int, int> *begin() const { return data_.data(); } + // .end() only required for py::make_iterator(self) overload + const std::pair<int, int> *end() const { return data_.data() + data_.size(); } + private: std::vector<std::pair<int, int>> data_; }; py::class_<IntPairs>(m, "IntPairs") .def(py::init<std::vector<std::pair<int, int>>>()) - .def("nonzero", [](const IntPairs& s) { - return py::make_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel()); - }, py::keep_alive<0, 1>()) - .def("nonzero_keys", [](const IntPairs& s) { - return py::make_key_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel()); - }, py::keep_alive<0, 1>()) - ; - + .def( + "nonzero", + [](const IntPairs &s) { + return py::make_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), + NonZeroSentinel()); + }, + py::keep_alive<0, 1>()) + .def( + "nonzero_keys", + [](const IntPairs &s) { + return py::make_key_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), + NonZeroSentinel()); + }, + py::keep_alive<0, 1>()) + .def( + "nonzero_values", + [](const IntPairs &s) { + return py::make_value_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), + NonZeroSentinel()); + }, + py::keep_alive<0, 1>()) + + // test iterator that returns values instead of references + .def( + "nonref", + [](const IntPairs &s) { + return py::make_iterator(NonRefIterator<std::pair<int, int>>(s.begin()), + NonRefIterator<std::pair<int, int>>(s.end())); + }, + py::keep_alive<0, 1>()) + .def( + "nonref_keys", + [](const IntPairs &s) { + return py::make_key_iterator(NonRefIterator<std::pair<int, int>>(s.begin()), + NonRefIterator<std::pair<int, int>>(s.end())); + }, + py::keep_alive<0, 1>()) + .def( + "nonref_values", + [](const IntPairs &s) { + return py::make_value_iterator(NonRefIterator<std::pair<int, int>>(s.begin()), + NonRefIterator<std::pair<int, int>>(s.end())); + }, + py::keep_alive<0, 1>()) + + // test single-argument make_iterator + .def( + "simple_iterator", + [](IntPairs &self) { return py::make_iterator(self); }, + py::keep_alive<0, 1>()) + .def( + "simple_keys", + [](IntPairs &self) { return py::make_key_iterator(self); }, + py::keep_alive<0, 1>()) + .def( + "simple_values", + [](IntPairs &self) { return py::make_value_iterator(self); }, + py::keep_alive<0, 1>()) + + // Test iterator with an Extra (doesn't do anything useful, so not used + // at runtime, but tests need to be able to compile with the correct + // overload. See PR #3293. + .def( + "_make_iterator_extras", + [](IntPairs &self) { return py::make_iterator(self, py::call_guard<int>()); }, + py::keep_alive<0, 1>()) + .def( + "_make_key_extras", + [](IntPairs &self) { return py::make_key_iterator(self, py::call_guard<int>()); }, + py::keep_alive<0, 1>()) + .def( + "_make_value_extras", + [](IntPairs &self) { return py::make_value_iterator(self, py::call_guard<int>()); }, + py::keep_alive<0, 1>()); + + // test_iterator_referencing + py::class_<NonCopyableInt>(m, "NonCopyableInt") + .def(py::init<int>()) + .def("set", &NonCopyableInt::set) + .def("__int__", &NonCopyableInt::get); + py::class_<std::vector<NonCopyableInt>>(m, "VectorNonCopyableInt") + .def(py::init<>()) + .def("append", + [](std::vector<NonCopyableInt> &vec, int value) { vec.emplace_back(value); }) + .def("__iter__", [](std::vector<NonCopyableInt> &vec) { + return py::make_iterator(vec.begin(), vec.end()); + }); + py::class_<std::vector<NonCopyableIntPair>>(m, "VectorNonCopyableIntPair") + .def(py::init<>()) + .def("append", + [](std::vector<NonCopyableIntPair> &vec, const std::pair<int, int> &value) { + vec.emplace_back(NonCopyableInt(value.first), NonCopyableInt(value.second)); + }) + .def("keys", + [](std::vector<NonCopyableIntPair> &vec) { + return py::make_key_iterator(vec.begin(), vec.end()); + }) + .def("values", [](std::vector<NonCopyableIntPair> &vec) { + return py::make_value_iterator(vec.begin(), vec.end()); + }); #if 0 // Obsolete: special data structure for exposing custom iterator types to python @@ -304,7 +506,7 @@ TEST_SUBMODULE(sequences_and_iterators, m) { #endif // test_python_iterator_in_cpp - m.def("object_to_list", [](py::object o) { + m.def("object_to_list", [](const py::object &o) { auto l = py::list(); for (auto item : o) { l.append(item); @@ -322,22 +524,22 @@ TEST_SUBMODULE(sequences_and_iterators, m) { }); // test_sequence_length: check that Python sequences can be converted to py::sequence. - m.def("sequence_length", [](py::sequence seq) { return seq.size(); }); + m.def("sequence_length", [](const py::sequence &seq) { return seq.size(); }); // Make sure that py::iterator works with std algorithms - m.def("count_none", [](py::object o) { + m.def("count_none", [](const py::object &o) { return std::count_if(o.begin(), o.end(), [](py::handle h) { return h.is_none(); }); }); - m.def("find_none", [](py::object o) { + m.def("find_none", [](const py::object &o) { auto it = std::find_if(o.begin(), o.end(), [](py::handle h) { return h.is_none(); }); return it->is_none(); }); - m.def("count_nonzeros", [](py::dict d) { - return std::count_if(d.begin(), d.end(), [](std::pair<py::handle, py::handle> p) { - return p.second.cast<int>() != 0; - }); + m.def("count_nonzeros", [](const py::dict &d) { + return std::count_if(d.begin(), d.end(), [](std::pair<py::handle, py::handle> p) { + return p.second.cast<int>() != 0; + }); }); m.def("tuple_iterator", &test_random_access_iterator<py::tuple>); @@ -352,7 +554,28 @@ TEST_SUBMODULE(sequences_and_iterators, m) { // test_iterator_rvp // #388: Can't make iterators via make_iterator() with different r/v policies - static std::vector<int> list = { 1, 2, 3 }; - m.def("make_iterator_1", []() { return py::make_iterator<py::return_value_policy::copy>(list); }); - m.def("make_iterator_2", []() { return py::make_iterator<py::return_value_policy::automatic>(list); }); + static std::vector<int> list = {1, 2, 3}; + m.def("make_iterator_1", + []() { return py::make_iterator<py::return_value_policy::copy>(list); }); + m.def("make_iterator_2", + []() { return py::make_iterator<py::return_value_policy::automatic>(list); }); + + // test_iterator on c arrays + // #4100: ensure lvalue required as increment operand + class CArrayHolder { + public: + CArrayHolder(double x, double y, double z) { + values[0] = x; + values[1] = y; + values[2] = z; + }; + double values[3]; + }; + + py::class_<CArrayHolder>(m, "CArrayHolder") + .def(py::init<double, double, double>()) + .def( + "__iter__", + [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); }, + py::keep_alive<0, 1>()); } diff --git a/tests/test_sequences_and_iterators.py b/tests/test_sequences_and_iterators.py index c3b608c4..dc129f2b 100644 --- a/tests/test_sequences_and_iterators.py +++ b/tests/test_sequences_and_iterators.py @@ -1,18 +1,19 @@ -# -*- coding: utf-8 -*- import pytest -from pybind11_tests import sequences_and_iterators as m +from pytest import approx # noqa: PT013 + from pybind11_tests import ConstructorStats +from pybind11_tests import sequences_and_iterators as m -def isclose(a, b, rel_tol=1e-05, abs_tol=0.0): - """Like math.isclose() from Python 3.5""" - return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) +def test_slice_constructors(): + assert m.make_forward_slice_size_t() == slice(0, -1, 1) + assert m.make_reversed_slice_object() == slice(None, None, -1) -def allclose(a_list, b_list, rel_tol=1e-05, abs_tol=0.0): - return all( - isclose(a, b, rel_tol=rel_tol, abs_tol=abs_tol) for a, b in zip(a_list, b_list) - ) +@pytest.mark.skipif(not m.has_optional, reason="no <optional>") +def test_slice_constructors_explicit_optional(): + assert m.make_reversed_slice_size_t_optional() == slice(None, None, -1) + assert m.make_reversed_slice_size_t_optional_verbose() == slice(None, None, -1) def test_generalized_iterators(): @@ -24,6 +25,10 @@ def test_generalized_iterators(): assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero_keys()) == [1] assert list(m.IntPairs([(0, 3), (1, 2), (3, 4)]).nonzero_keys()) == [] + assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).nonzero_values()) == [2, 4] + assert list(m.IntPairs([(1, 2), (2, 0), (0, 3), (4, 5)]).nonzero_values()) == [2] + assert list(m.IntPairs([(0, 3), (1, 2), (3, 4)]).nonzero_values()) == [] + # __next__ must continue to raise StopIteration it = m.IntPairs([(0, 0)]).nonzero() for _ in range(3): @@ -36,6 +41,47 @@ def test_generalized_iterators(): next(it) +def test_nonref_iterators(): + pairs = m.IntPairs([(1, 2), (3, 4), (0, 5)]) + assert list(pairs.nonref()) == [(1, 2), (3, 4), (0, 5)] + assert list(pairs.nonref_keys()) == [1, 3, 0] + assert list(pairs.nonref_values()) == [2, 4, 5] + + +def test_generalized_iterators_simple(): + assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_iterator()) == [ + (1, 2), + (3, 4), + (0, 5), + ] + assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_keys()) == [1, 3, 0] + assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_values()) == [2, 4, 5] + + +def test_iterator_referencing(): + """Test that iterators reference rather than copy their referents.""" + vec = m.VectorNonCopyableInt() + vec.append(3) + vec.append(5) + assert [int(x) for x in vec] == [3, 5] + # Increment everything to make sure the referents can be mutated + for x in vec: + x.set(int(x) + 1) + assert [int(x) for x in vec] == [4, 6] + + vec = m.VectorNonCopyableIntPair() + vec.append([3, 4]) + vec.append([5, 7]) + assert [int(x) for x in vec.keys()] == [3, 5] + assert [int(x) for x in vec.values()] == [4, 7] + for x in vec.keys(): + x.set(int(x) + 1) + for x in vec.values(): + x.set(int(x) + 10) + assert [int(x) for x in vec.keys()] == [4, 6] + assert [int(x) for x in vec.values()] == [14, 17] + + def test_sliceable(): sliceable = m.Sliceable(100) assert sliceable[::] == (0, 100, 1) @@ -57,11 +103,13 @@ def test_sequence(): assert "Sequence" in repr(s) assert len(s) == 5 - assert s[0] == 0 and s[3] == 0 + assert s[0] == 0 + assert s[3] == 0 assert 12.34 not in s s[0], s[3] = 12.34, 56.78 assert 12.34 in s - assert isclose(s[0], 12.34) and isclose(s[3], 56.78) + assert s[0] == approx(12.34, rel=1e-05) + assert s[3] == approx(56.78, rel=1e-05) rev = reversed(s) assert cstats.values() == ["of size", "5"] @@ -76,14 +124,14 @@ def test_sequence(): assert cstats.values() == ["of size", "0"] expected = [0, 56.78, 0, 0, 12.34] - assert allclose(rev, expected) - assert allclose(rev2, expected) + assert rev == approx(expected, rel=1e-05) + assert rev2 == approx(expected, rel=1e-05) assert rev == rev2 rev[0::2] = m.Sequence([2.0, 2.0, 2.0]) assert cstats.values() == ["of size", "3", "from std::vector"] - assert allclose(rev, [2, 56.78, 2, 0, 2]) + assert rev == approx([2, 56.78, 2, 0, 2], rel=1e-05) assert cstats.alive() == 4 del it @@ -104,7 +152,7 @@ def test_sequence(): def test_sequence_length(): - """#2076: Exception raised by len(arg) should be propagated """ + """#2076: Exception raised by len(arg) should be propagated""" class BadLen(RuntimeError): pass @@ -139,6 +187,7 @@ def test_map_iterator(): assert sm[k] == expected[k] for k, v in sm.items(): assert v == expected[k] + assert list(sm.values()) == [expected[k] for k in sm] it = iter(m.StringMap({})) for _ in range(3): # __next__ must continue to raise StopIteration @@ -187,9 +236,17 @@ def test_iterator_passthrough(): def test_iterator_rvp(): - """#388: Can't make iterators via make_iterator() with different r/v policies """ + """#388: Can't make iterators via make_iterator() with different r/v policies""" import pybind11_tests.sequences_and_iterators as m assert list(m.make_iterator_1()) == [1, 2, 3] assert list(m.make_iterator_2()) == [1, 2, 3] assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2())) + + +def test_carray_iterator(): + """#4100: Check for proper iterator overload with C-Arrays""" + args_gt = [float(i) for i in range(3)] + arr_h = m.CArrayHolder(*args_gt) + args = list(arr_h) + assert args_gt == args diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp index 59996ede..6d9efced 100644 --- a/tests/test_smart_ptr.cpp +++ b/tests/test_smart_ptr.cpp @@ -8,53 +8,34 @@ BSD-style license that can be found in the LICENSE file. */ -#if defined(_MSC_VER) && _MSC_VER < 1910 // VS 2015's MSVC -# pragma warning(disable: 4702) // unreachable code in system header (xatomic.h(382)) -#endif - -#include "pybind11_tests.h" #include "object.h" +#include "pybind11_tests.h" -// Make pybind aware of the ref-counted wrapper type (s): - -// ref<T> is a wrapper for 'Object' which uses intrusive reference counting -// It is always possible to construct a ref<T> from an Object* pointer without -// possible inconsistencies, hence the 'true' argument at the end. -PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true); -// Make pybind11 aware of the non-standard getter member function -namespace pybind11 { namespace detail { - template <typename T> - struct holder_helper<ref<T>> { - static const T *get(const ref<T> &p) { return p.get_ptr(); } - }; -} // namespace detail -} // namespace pybind11 - -// The following is not required anymore for std::shared_ptr, but it should compile without error: -PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>); +namespace { // This is just a wrapper around unique_ptr, but with extra fields to deliberately bloat up the -// holder size to trigger the non-simple-layout internal instance layout for single inheritance with -// large holder type: -template <typename T> class huge_unique_ptr { +// holder size to trigger the non-simple-layout internal instance layout for single inheritance +// with large holder type: +template <typename T> +class huge_unique_ptr { std::unique_ptr<T> ptr; uint64_t padding[10]; + public: - huge_unique_ptr(T *p) : ptr(p) {}; + explicit huge_unique_ptr(T *p) : ptr(p) {} T *get() { return ptr.get(); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr<T>); // Simple custom holder that works like unique_ptr template <typename T> class custom_unique_ptr { std::unique_ptr<T> impl; + public: - custom_unique_ptr(T* p) : impl(p) { } - T* get() const { return impl.get(); } - T* release_ptr() { return impl.release(); } + explicit custom_unique_ptr(T *p) : impl(p) {} + T *get() const { return impl.get(); } + T *release_ptr() { return impl.release(); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr<T>); // Simple custom holder that works like shared_ptr and has operator& overload // To obtain address of an instance of this holder pybind should use std::addressof @@ -62,13 +43,13 @@ PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr<T>); template <typename T> class shared_ptr_with_addressof_operator { std::shared_ptr<T> impl; + public: - shared_ptr_with_addressof_operator( ) = default; - shared_ptr_with_addressof_operator(T* p) : impl(p) { } - T* get() const { return impl.get(); } - T** operator&() { throw std::logic_error("Call of overloaded operator& is not expected"); } + shared_ptr_with_addressof_operator() = default; + explicit shared_ptr_with_addressof_operator(T *p) : impl(p) {} + T *get() const { return impl.get(); } + T **operator&() { throw std::logic_error("Call of overloaded operator& is not expected"); } }; -PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator<T>); // Simple custom holder that works like unique_ptr and has operator& overload // To obtain address of an instance of this holder pybind should use std::addressof @@ -76,17 +57,237 @@ PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator<T>); template <typename T> class unique_ptr_with_addressof_operator { std::unique_ptr<T> impl; + public: unique_ptr_with_addressof_operator() = default; - unique_ptr_with_addressof_operator(T* p) : impl(p) { } - T* get() const { return impl.get(); } - T* release_ptr() { return impl.release(); } - T** operator&() { throw std::logic_error("Call of overloaded operator& is not expected"); } + explicit unique_ptr_with_addressof_operator(T *p) : impl(p) {} + T *get() const { return impl.get(); } + T *release_ptr() { return impl.release(); } + T **operator&() { throw std::logic_error("Call of overloaded operator& is not expected"); } +}; + +// Custom object with builtin reference counting (see 'object.h' for the implementation) +class MyObject1 : public Object { +public: + explicit MyObject1(int value) : value(value) { print_created(this, toString()); } + std::string toString() const override { return "MyObject1[" + std::to_string(value) + "]"; } + +protected: + ~MyObject1() override { print_destroyed(this); } + +private: + int value; +}; + +// Object managed by a std::shared_ptr<> +class MyObject2 { +public: + MyObject2(const MyObject2 &) = default; + explicit MyObject2(int value) : value(value) { print_created(this, toString()); } + std::string toString() const { return "MyObject2[" + std::to_string(value) + "]"; } + virtual ~MyObject2() { print_destroyed(this); } + +private: + int value; +}; + +// Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<> +class MyObject3 : public std::enable_shared_from_this<MyObject3> { +public: + MyObject3(const MyObject3 &) = default; + explicit MyObject3(int value) : value(value) { print_created(this, toString()); } + std::string toString() const { return "MyObject3[" + std::to_string(value) + "]"; } + virtual ~MyObject3() { print_destroyed(this); } + +private: + int value; +}; + +// test_unique_nodelete +// Object with a private destructor +class MyObject4; +std::unordered_set<MyObject4 *> myobject4_instances; +class MyObject4 { +public: + explicit MyObject4(int value) : value{value} { + print_created(this); + myobject4_instances.insert(this); + } + int value; + + static void cleanupAllInstances() { + auto tmp = std::move(myobject4_instances); + myobject4_instances.clear(); + for (auto *o : tmp) { + delete o; + } + } + +private: + ~MyObject4() { + myobject4_instances.erase(this); + print_destroyed(this); + } +}; + +// test_unique_deleter +// Object with std::unique_ptr<T, D> where D is not matching the base class +// Object with a protected destructor +class MyObject4a; +std::unordered_set<MyObject4a *> myobject4a_instances; +class MyObject4a { +public: + explicit MyObject4a(int i) : value{i} { + print_created(this); + myobject4a_instances.insert(this); + }; + int value; + + static void cleanupAllInstances() { + auto tmp = std::move(myobject4a_instances); + myobject4a_instances.clear(); + for (auto *o : tmp) { + delete o; + } + } + +protected: + virtual ~MyObject4a() { + myobject4a_instances.erase(this); + print_destroyed(this); + } +}; + +// Object derived but with public destructor and no Deleter in default holder +class MyObject4b : public MyObject4a { +public: + explicit MyObject4b(int i) : MyObject4a(i) { print_created(this); } + ~MyObject4b() override { print_destroyed(this); } +}; + +// test_large_holder +class MyObject5 { // managed by huge_unique_ptr +public: + explicit MyObject5(int value) : value{value} { print_created(this); } + ~MyObject5() { print_destroyed(this); } + int value; +}; + +// test_shared_ptr_and_references +struct SharedPtrRef { + struct A { + A() { print_created(this); } + A(const A &) { print_copy_created(this); } + A(A &&) noexcept { print_move_created(this); } + ~A() { print_destroyed(this); } + }; + + A value = {}; + std::shared_ptr<A> shared = std::make_shared<A>(); +}; + +// test_shared_ptr_from_this_and_references +struct SharedFromThisRef { + struct B : std::enable_shared_from_this<B> { + B() { print_created(this); } + // NOLINTNEXTLINE(bugprone-copy-constructor-init) + B(const B &) : std::enable_shared_from_this<B>() { print_copy_created(this); } + B(B &&) noexcept : std::enable_shared_from_this<B>() { print_move_created(this); } + ~B() { print_destroyed(this); } + }; + + B value = {}; + std::shared_ptr<B> shared = std::make_shared<B>(); +}; + +// Issue #865: shared_from_this doesn't work with virtual inheritance +struct SharedFromThisVBase : std::enable_shared_from_this<SharedFromThisVBase> { + SharedFromThisVBase() = default; + SharedFromThisVBase(const SharedFromThisVBase &) = default; + virtual ~SharedFromThisVBase() = default; +}; +struct SharedFromThisVirt : virtual SharedFromThisVBase {}; + +// test_move_only_holder +struct C { + C() { print_created(this); } + ~C() { print_destroyed(this); } +}; + +// test_holder_with_addressof_operator +struct TypeForHolderWithAddressOf { + TypeForHolderWithAddressOf() { print_created(this); } + TypeForHolderWithAddressOf(const TypeForHolderWithAddressOf &) { print_copy_created(this); } + TypeForHolderWithAddressOf(TypeForHolderWithAddressOf &&) noexcept { + print_move_created(this); + } + ~TypeForHolderWithAddressOf() { print_destroyed(this); } + std::string toString() const { + return "TypeForHolderWithAddressOf[" + std::to_string(value) + "]"; + } + int value = 42; +}; + +// test_move_only_holder_with_addressof_operator +struct TypeForMoveOnlyHolderWithAddressOf { + explicit TypeForMoveOnlyHolderWithAddressOf(int value) : value{value} { print_created(this); } + ~TypeForMoveOnlyHolderWithAddressOf() { print_destroyed(this); } + std::string toString() const { + return "MoveOnlyHolderWithAddressOf[" + std::to_string(value) + "]"; + } + int value; +}; + +// test_smart_ptr_from_default +struct HeldByDefaultHolder {}; + +// test_shared_ptr_gc +// #187: issue involving std::shared_ptr<> return value policy & garbage collection +struct ElementBase { + virtual ~ElementBase() = default; /* Force creation of virtual table */ + ElementBase() = default; + ElementBase(const ElementBase &) = delete; +}; + +struct ElementA : ElementBase { + explicit ElementA(int v) : v(v) {} + int value() const { return v; } + int v; +}; + +struct ElementList { + void add(const std::shared_ptr<ElementBase> &e) { l.push_back(e); } + std::vector<std::shared_ptr<ElementBase>> l; }; -PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator<T>); +} // namespace + +// ref<T> is a wrapper for 'Object' which uses intrusive reference counting +// It is always possible to construct a ref<T> from an Object* pointer without +// possible inconsistencies, hence the 'true' argument at the end. +// Make pybind11 aware of the non-standard getter member function +namespace PYBIND11_NAMESPACE { +namespace detail { +template <typename T> +struct holder_helper<ref<T>> { + static const T *get(const ref<T> &p) { return p.get_ptr(); } +}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE + +// Make pybind aware of the ref-counted wrapper type (s): +PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true); +// The following is not required anymore for std::shared_ptr, but it should compile without error: +PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>); +PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr<T>); +PYBIND11_DECLARE_HOLDER_TYPE(T, custom_unique_ptr<T>); +PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_with_addressof_operator<T>); +PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator<T>); TEST_SUBMODULE(smart_ptr, m) { + // Please do not interleave `struct` and `class` definitions with bindings code, + // but implement `struct`s and `class`es in the anonymous namespace above. + // This helps keeping the smart_holder branch in sync with master. // test_smart_ptr @@ -94,24 +295,13 @@ TEST_SUBMODULE(smart_ptr, m) { py::class_<Object, ref<Object>> obj(m, "Object"); obj.def("getRefCount", &Object::getRefCount); - // Custom object with builtin reference counting (see 'object.h' for the implementation) - class MyObject1 : public Object { - public: - MyObject1(int value) : value(value) { print_created(this, toString()); } - std::string toString() const override { return "MyObject1[" + std::to_string(value) + "]"; } - protected: - ~MyObject1() override { print_destroyed(this); } - private: - int value; - }; - py::class_<MyObject1, ref<MyObject1>>(m, "MyObject1", obj) - .def(py::init<int>()); + py::class_<MyObject1, ref<MyObject1>>(m, "MyObject1", obj).def(py::init<int>()); py::implicitly_convertible<py::int_, MyObject1>(); m.def("make_object_1", []() -> Object * { return new MyObject1(1); }); - m.def("make_object_2", []() -> ref<Object> { return new MyObject1(2); }); + m.def("make_object_2", []() -> ref<Object> { return ref<Object>(new MyObject1(2)); }); m.def("make_myobject1_1", []() -> MyObject1 * { return new MyObject1(4); }); - m.def("make_myobject1_2", []() -> ref<MyObject1> { return new MyObject1(5); }); + m.def("make_myobject1_2", []() -> ref<MyObject1> { return ref<MyObject1>(new MyObject1(5)); }); m.def("print_object_1", [](const Object *obj) { py::print(obj->toString()); }); m.def("print_object_2", [](ref<Object> obj) { py::print(obj->toString()); }); m.def("print_object_3", [](const ref<Object> &obj) { py::print(obj->toString()); }); @@ -124,48 +314,31 @@ TEST_SUBMODULE(smart_ptr, m) { // Expose constructor stats for the ref type m.def("cstats_ref", &ConstructorStats::get<ref_tag>); - - // Object managed by a std::shared_ptr<> - class MyObject2 { - public: - MyObject2(const MyObject2 &) = default; - MyObject2(int value) : value(value) { print_created(this, toString()); } - std::string toString() const { return "MyObject2[" + std::to_string(value) + "]"; } - virtual ~MyObject2() { print_destroyed(this); } - private: - int value; - }; - py::class_<MyObject2, std::shared_ptr<MyObject2>>(m, "MyObject2") - .def(py::init<int>()); + py::class_<MyObject2, std::shared_ptr<MyObject2>>(m, "MyObject2").def(py::init<int>()); m.def("make_myobject2_1", []() { return new MyObject2(6); }); m.def("make_myobject2_2", []() { return std::make_shared<MyObject2>(7); }); m.def("print_myobject2_1", [](const MyObject2 *obj) { py::print(obj->toString()); }); + // NOLINTNEXTLINE(performance-unnecessary-value-param) m.def("print_myobject2_2", [](std::shared_ptr<MyObject2> obj) { py::print(obj->toString()); }); - m.def("print_myobject2_3", [](const std::shared_ptr<MyObject2> &obj) { py::print(obj->toString()); }); - m.def("print_myobject2_4", [](const std::shared_ptr<MyObject2> *obj) { py::print((*obj)->toString()); }); - - // Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<> - class MyObject3 : public std::enable_shared_from_this<MyObject3> { - public: - MyObject3(const MyObject3 &) = default; - MyObject3(int value) : value(value) { print_created(this, toString()); } - std::string toString() const { return "MyObject3[" + std::to_string(value) + "]"; } - virtual ~MyObject3() { print_destroyed(this); } - private: - int value; - }; - py::class_<MyObject3, std::shared_ptr<MyObject3>>(m, "MyObject3") - .def(py::init<int>()); + m.def("print_myobject2_3", + [](const std::shared_ptr<MyObject2> &obj) { py::print(obj->toString()); }); + m.def("print_myobject2_4", + [](const std::shared_ptr<MyObject2> *obj) { py::print((*obj)->toString()); }); + + py::class_<MyObject3, std::shared_ptr<MyObject3>>(m, "MyObject3").def(py::init<int>()); m.def("make_myobject3_1", []() { return new MyObject3(8); }); m.def("make_myobject3_2", []() { return std::make_shared<MyObject3>(9); }); m.def("print_myobject3_1", [](const MyObject3 *obj) { py::print(obj->toString()); }); + // NOLINTNEXTLINE(performance-unnecessary-value-param) m.def("print_myobject3_2", [](std::shared_ptr<MyObject3> obj) { py::print(obj->toString()); }); - m.def("print_myobject3_3", [](const std::shared_ptr<MyObject3> &obj) { py::print(obj->toString()); }); - m.def("print_myobject3_4", [](const std::shared_ptr<MyObject3> *obj) { py::print((*obj)->toString()); }); + m.def("print_myobject3_3", + [](const std::shared_ptr<MyObject3> &obj) { py::print(obj->toString()); }); + m.def("print_myobject3_4", + [](const std::shared_ptr<MyObject3> *obj) { py::print((*obj)->toString()); }); // test_smart_ptr_refcounting m.def("test_object1_refcounting", []() { - ref<MyObject1> o = new MyObject1(0); + auto o = ref<MyObject1>(new MyObject1(0)); bool good = o->getRefCount() == 1; py::object o2 = py::cast(o, py::return_value_policy::reference); // always request (partial) ownership for objects with intrusive @@ -175,226 +348,123 @@ TEST_SUBMODULE(smart_ptr, m) { }); // test_unique_nodelete - // Object with a private destructor - class MyObject4; - static std::unordered_set<MyObject4 *> myobject4_instances; - class MyObject4 { - public: - MyObject4(int value) : value{value} { - print_created(this); - myobject4_instances.insert(this); - } - int value; - - static void cleanupAllInstances() { - auto tmp = std::move(myobject4_instances); - myobject4_instances.clear(); - for (auto o : tmp) - delete o; - } - private: - ~MyObject4() { - myobject4_instances.erase(this); - print_destroyed(this); - } - }; py::class_<MyObject4, std::unique_ptr<MyObject4, py::nodelete>>(m, "MyObject4") .def(py::init<int>()) .def_readwrite("value", &MyObject4::value) .def_static("cleanup_all_instances", &MyObject4::cleanupAllInstances); // test_unique_deleter - // Object with std::unique_ptr<T, D> where D is not matching the base class - // Object with a protected destructor - class MyObject4a; - static std::unordered_set<MyObject4a *> myobject4a_instances; - class MyObject4a { - public: - MyObject4a(int i) { - value = i; - print_created(this); - myobject4a_instances.insert(this); - }; - int value; - - static void cleanupAllInstances() { - auto tmp = std::move(myobject4a_instances); - myobject4a_instances.clear(); - for (auto o : tmp) - delete o; - } - protected: - virtual ~MyObject4a() { - myobject4a_instances.erase(this); - print_destroyed(this); - } - }; py::class_<MyObject4a, std::unique_ptr<MyObject4a, py::nodelete>>(m, "MyObject4a") .def(py::init<int>()) .def_readwrite("value", &MyObject4a::value) .def_static("cleanup_all_instances", &MyObject4a::cleanupAllInstances); - // Object derived but with public destructor and no Deleter in default holder - class MyObject4b : public MyObject4a { - public: - MyObject4b(int i) : MyObject4a(i) { print_created(this); } - ~MyObject4b() override { print_destroyed(this); } - }; - py::class_<MyObject4b, MyObject4a>(m, "MyObject4b") + py::class_<MyObject4b, MyObject4a, std::unique_ptr<MyObject4b>>(m, "MyObject4b") .def(py::init<int>()); // test_large_holder - class MyObject5 { // managed by huge_unique_ptr - public: - MyObject5(int value) : value{value} { print_created(this); } - ~MyObject5() { print_destroyed(this); } - int value; - }; py::class_<MyObject5, huge_unique_ptr<MyObject5>>(m, "MyObject5") .def(py::init<int>()) .def_readwrite("value", &MyObject5::value); // test_shared_ptr_and_references - struct SharedPtrRef { - struct A { - A() { print_created(this); } - A(const A &) { print_copy_created(this); } - A(A &&) { print_move_created(this); } - ~A() { print_destroyed(this); } - }; - - A value = {}; - std::shared_ptr<A> shared = std::make_shared<A>(); - }; using A = SharedPtrRef::A; py::class_<A, std::shared_ptr<A>>(m, "A"); - py::class_<SharedPtrRef>(m, "SharedPtrRef") + py::class_<SharedPtrRef, std::unique_ptr<SharedPtrRef>>(m, "SharedPtrRef") .def(py::init<>()) .def_readonly("ref", &SharedPtrRef::value) - .def_property_readonly("copy", [](const SharedPtrRef &s) { return s.value; }, - py::return_value_policy::copy) + .def_property_readonly( + "copy", [](const SharedPtrRef &s) { return s.value; }, py::return_value_policy::copy) .def_readonly("holder_ref", &SharedPtrRef::shared) - .def_property_readonly("holder_copy", [](const SharedPtrRef &s) { return s.shared; }, - py::return_value_policy::copy) + .def_property_readonly( + "holder_copy", + [](const SharedPtrRef &s) { return s.shared; }, + py::return_value_policy::copy) .def("set_ref", [](SharedPtrRef &, const A &) { return true; }) + // NOLINTNEXTLINE(performance-unnecessary-value-param) .def("set_holder", [](SharedPtrRef &, std::shared_ptr<A>) { return true; }); // test_shared_ptr_from_this_and_references - struct SharedFromThisRef { - struct B : std::enable_shared_from_this<B> { - B() { print_created(this); } - B(const B &) : std::enable_shared_from_this<B>() { print_copy_created(this); } - B(B &&) : std::enable_shared_from_this<B>() { print_move_created(this); } - ~B() { print_destroyed(this); } - }; - - B value = {}; - std::shared_ptr<B> shared = std::make_shared<B>(); - }; using B = SharedFromThisRef::B; py::class_<B, std::shared_ptr<B>>(m, "B"); - py::class_<SharedFromThisRef>(m, "SharedFromThisRef") + py::class_<SharedFromThisRef, std::unique_ptr<SharedFromThisRef>>(m, "SharedFromThisRef") .def(py::init<>()) .def_readonly("bad_wp", &SharedFromThisRef::value) - .def_property_readonly("ref", [](const SharedFromThisRef &s) -> const B & { return *s.shared; }) - .def_property_readonly("copy", [](const SharedFromThisRef &s) { return s.value; }, - py::return_value_policy::copy) + .def_property_readonly("ref", + [](const SharedFromThisRef &s) -> const B & { return *s.shared; }) + .def_property_readonly( + "copy", + [](const SharedFromThisRef &s) { return s.value; }, + py::return_value_policy::copy) .def_readonly("holder_ref", &SharedFromThisRef::shared) - .def_property_readonly("holder_copy", [](const SharedFromThisRef &s) { return s.shared; }, - py::return_value_policy::copy) + .def_property_readonly( + "holder_copy", + [](const SharedFromThisRef &s) { return s.shared; }, + py::return_value_policy::copy) .def("set_ref", [](SharedFromThisRef &, const B &) { return true; }) + // NOLINTNEXTLINE(performance-unnecessary-value-param) .def("set_holder", [](SharedFromThisRef &, std::shared_ptr<B>) { return true; }); // Issue #865: shared_from_this doesn't work with virtual inheritance - struct SharedFromThisVBase : std::enable_shared_from_this<SharedFromThisVBase> { - SharedFromThisVBase() = default; - SharedFromThisVBase(const SharedFromThisVBase &) = default; - virtual ~SharedFromThisVBase() = default; - }; - struct SharedFromThisVirt : virtual SharedFromThisVBase {}; static std::shared_ptr<SharedFromThisVirt> sft(new SharedFromThisVirt()); py::class_<SharedFromThisVirt, std::shared_ptr<SharedFromThisVirt>>(m, "SharedFromThisVirt") .def_static("get", []() { return sft.get(); }); // test_move_only_holder - struct C { - C() { print_created(this); } - ~C() { print_destroyed(this); } - }; py::class_<C, custom_unique_ptr<C>>(m, "TypeWithMoveOnlyHolder") .def_static("make", []() { return custom_unique_ptr<C>(new C); }) .def_static("make_as_object", []() { return py::cast(custom_unique_ptr<C>(new C)); }); // test_holder_with_addressof_operator - struct TypeForHolderWithAddressOf { - TypeForHolderWithAddressOf() { print_created(this); } - TypeForHolderWithAddressOf(const TypeForHolderWithAddressOf &) { print_copy_created(this); } - TypeForHolderWithAddressOf(TypeForHolderWithAddressOf &&) { print_move_created(this); } - ~TypeForHolderWithAddressOf() { print_destroyed(this); } - std::string toString() const { - return "TypeForHolderWithAddressOf[" + std::to_string(value) + "]"; - } - int value = 42; - }; using HolderWithAddressOf = shared_ptr_with_addressof_operator<TypeForHolderWithAddressOf>; py::class_<TypeForHolderWithAddressOf, HolderWithAddressOf>(m, "TypeForHolderWithAddressOf") .def_static("make", []() { return HolderWithAddressOf(new TypeForHolderWithAddressOf); }) .def("get", [](const HolderWithAddressOf &self) { return self.get(); }) - .def("print_object_1", [](const TypeForHolderWithAddressOf *obj) { py::print(obj->toString()); }) + .def("print_object_1", + [](const TypeForHolderWithAddressOf *obj) { py::print(obj->toString()); }) + // NOLINTNEXTLINE(performance-unnecessary-value-param) .def("print_object_2", [](HolderWithAddressOf obj) { py::print(obj.get()->toString()); }) - .def("print_object_3", [](const HolderWithAddressOf &obj) { py::print(obj.get()->toString()); }) - .def("print_object_4", [](const HolderWithAddressOf *obj) { py::print((*obj).get()->toString()); }); + .def("print_object_3", + [](const HolderWithAddressOf &obj) { py::print(obj.get()->toString()); }) + .def("print_object_4", + [](const HolderWithAddressOf *obj) { py::print((*obj).get()->toString()); }); // test_move_only_holder_with_addressof_operator - struct TypeForMoveOnlyHolderWithAddressOf { - TypeForMoveOnlyHolderWithAddressOf(int value) : value{value} { print_created(this); } - ~TypeForMoveOnlyHolderWithAddressOf() { print_destroyed(this); } - std::string toString() const { - return "MoveOnlyHolderWithAddressOf[" + std::to_string(value) + "]"; - } - int value; - }; - using MoveOnlyHolderWithAddressOf = unique_ptr_with_addressof_operator<TypeForMoveOnlyHolderWithAddressOf>; - py::class_<TypeForMoveOnlyHolderWithAddressOf, MoveOnlyHolderWithAddressOf>(m, "TypeForMoveOnlyHolderWithAddressOf") - .def_static("make", []() { return MoveOnlyHolderWithAddressOf(new TypeForMoveOnlyHolderWithAddressOf(0)); }) + using MoveOnlyHolderWithAddressOf + = unique_ptr_with_addressof_operator<TypeForMoveOnlyHolderWithAddressOf>; + py::class_<TypeForMoveOnlyHolderWithAddressOf, MoveOnlyHolderWithAddressOf>( + m, "TypeForMoveOnlyHolderWithAddressOf") + .def_static("make", + []() { + return MoveOnlyHolderWithAddressOf( + new TypeForMoveOnlyHolderWithAddressOf(0)); + }) .def_readwrite("value", &TypeForMoveOnlyHolderWithAddressOf::value) - .def("print_object", [](const TypeForMoveOnlyHolderWithAddressOf *obj) { py::print(obj->toString()); }); + .def("print_object", + [](const TypeForMoveOnlyHolderWithAddressOf *obj) { py::print(obj->toString()); }); // test_smart_ptr_from_default - struct HeldByDefaultHolder { }; - py::class_<HeldByDefaultHolder>(m, "HeldByDefaultHolder") + py::class_<HeldByDefaultHolder, std::unique_ptr<HeldByDefaultHolder>>(m, "HeldByDefaultHolder") .def(py::init<>()) + // NOLINTNEXTLINE(performance-unnecessary-value-param) .def_static("load_shared_ptr", [](std::shared_ptr<HeldByDefaultHolder>) {}); // test_shared_ptr_gc // #187: issue involving std::shared_ptr<> return value policy & garbage collection - struct ElementBase { - virtual ~ElementBase() = default; /* Force creation of virtual table */ - ElementBase() = default; - ElementBase(const ElementBase&) = delete; - }; py::class_<ElementBase, std::shared_ptr<ElementBase>>(m, "ElementBase"); - struct ElementA : ElementBase { - ElementA(int v) : v(v) { } - int value() { return v; } - int v; - }; py::class_<ElementA, ElementBase, std::shared_ptr<ElementA>>(m, "ElementA") .def(py::init<int>()) .def("value", &ElementA::value); - struct ElementList { - void add(std::shared_ptr<ElementBase> e) { l.push_back(e); } - std::vector<std::shared_ptr<ElementBase>> l; - }; py::class_<ElementList, std::shared_ptr<ElementList>>(m, "ElementList") .def(py::init<>()) .def("add", &ElementList::add) .def("get", [](ElementList &el) { py::list list; - for (auto &e : el.l) + for (auto &e : el.l) { list.append(py::cast(e)); + } return list; }); } diff --git a/tests/test_smart_ptr.py b/tests/test_smart_ptr.py index 85f61a32..2f204e01 100644 --- a/tests/test_smart_ptr.py +++ b/tests/test_smart_ptr.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import pytest m = pytest.importorskip("pybind11_tests.smart_ptr") @@ -16,7 +15,7 @@ def test_smart_ptr(capture): m.print_object_2(o) m.print_object_3(o) m.print_object_4(o) - assert capture == "MyObject1[{i}]\n".format(i=i) * 4 + assert capture == f"MyObject1[{i}]\n" * 4 for i, o in enumerate( [m.make_myobject1_1(), m.make_myobject1_2(), m.MyObject1(6), 7], start=4 @@ -34,13 +33,11 @@ def test_smart_ptr(capture): m.print_myobject1_4(o) times = 4 if isinstance(o, int) else 8 - assert capture == "MyObject1[{i}]\n".format(i=i) * times + assert capture == f"MyObject1[{i}]\n" * times cstats = ConstructorStats.get(m.MyObject1) assert cstats.alive() == 0 - expected_values = ["MyObject1[{}]".format(i) for i in range(1, 7)] + [ - "MyObject1[7]" - ] * 4 + expected_values = [f"MyObject1[{i}]" for i in range(1, 7)] + ["MyObject1[7]"] * 4 assert cstats.values() == expected_values assert cstats.default_constructions == 0 assert cstats.copy_constructions == 0 @@ -58,7 +55,7 @@ def test_smart_ptr(capture): m.print_myobject2_2(o) m.print_myobject2_3(o) m.print_myobject2_4(o) - assert capture == "MyObject2[{i}]\n".format(i=i) * 4 + assert capture == f"MyObject2[{i}]\n" * 4 cstats = ConstructorStats.get(m.MyObject2) assert cstats.alive() == 1 @@ -81,7 +78,7 @@ def test_smart_ptr(capture): m.print_myobject3_2(o) m.print_myobject3_3(o) m.print_myobject3_4(o) - assert capture == "MyObject3[{i}]\n".format(i=i) * 4 + assert capture == f"MyObject3[{i}]\n" * 4 cstats = ConstructorStats.get(m.MyObject3) assert cstats.alive() == 1 diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 05901627..d45465d6 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -7,22 +7,44 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/stl.h> -#include <vector> +#include "constructor_stats.h" +#include "pybind11_tests.h" + +#ifndef PYBIND11_HAS_FILESYSTEM_IS_OPTIONAL +# define PYBIND11_HAS_FILESYSTEM_IS_OPTIONAL +#endif +#include <pybind11/stl/filesystem.h> + #include <string> +#include <vector> + +#if defined(PYBIND11_TEST_BOOST) +# include <boost/optional.hpp> + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <typename T> +struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {}; + +template <> +struct type_caster<boost::none_t> : void_caster<boost::none_t> {}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE +#endif // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14 #if defined(PYBIND11_HAS_VARIANT) using std::variant; -#elif defined(PYBIND11_TEST_BOOST) && (!defined(_MSC_VER) || _MSC_VER >= 1910) -# include <boost/variant.hpp> -# define PYBIND11_HAS_VARIANT 1 +# define PYBIND11_TEST_VARIANT 1 +#elif defined(PYBIND11_TEST_BOOST) +# include <boost/variant.hpp> +# define PYBIND11_TEST_VARIANT 1 using boost::variant; -namespace pybind11 { namespace detail { +namespace PYBIND11_NAMESPACE { +namespace detail { template <typename... Ts> struct type_caster<boost::variant<Ts...>> : variant_caster<boost::variant<Ts...>> {}; @@ -33,33 +55,117 @@ struct visit_helper<boost::variant> { return boost::apply_visitor(args...); } }; -}} // namespace pybind11::detail +} // namespace detail +} // namespace PYBIND11_NAMESPACE #endif PYBIND11_MAKE_OPAQUE(std::vector<std::string, std::allocator<std::string>>); /// Issue #528: templated constructor struct TplCtorClass { - template <typename T> TplCtorClass(const T &) { } + template <typename T> + explicit TplCtorClass(const T &) {} bool operator==(const TplCtorClass &) const { return true; } }; namespace std { - template <> - struct hash<TplCtorClass> { size_t operator()(const TplCtorClass &) const { return 0; } }; +template <> +struct hash<TplCtorClass> { + size_t operator()(const TplCtorClass &) const { return 0; } +}; } // namespace std - template <template <typename> class OptionalImpl, typename T> -struct OptionalHolder -{ - OptionalHolder() = default; - bool member_initialized() const { - return member && member->initialized; - } +struct OptionalHolder { + // NOLINTNEXTLINE(modernize-use-equals-default): breaks GCC 4.8 + OptionalHolder(){}; + bool member_initialized() const { return member && member->initialized; } OptionalImpl<T> member = T{}; }; +enum class EnumType { + kSet = 42, + kUnset = 85, +}; + +// This is used to test that return-by-ref and return-by-copy policies are +// handled properly for optional types. This is a regression test for a dangling +// reference issue. The issue seemed to require the enum value type to +// reproduce - it didn't seem to happen if the value type is just an integer. +template <template <typename> class OptionalImpl> +class OptionalProperties { +public: + using OptionalEnumValue = OptionalImpl<EnumType>; + + OptionalProperties() : value(EnumType::kSet) {} + ~OptionalProperties() { + // Reset value to detect use-after-destruction. + // This is set to a specific value rather than nullopt to ensure that + // the memory that contains the value gets re-written. + value = EnumType::kUnset; + } + + OptionalEnumValue &access_by_ref() { return value; } + OptionalEnumValue access_by_copy() { return value; } + +private: + OptionalEnumValue value; +}; + +// This type mimics aspects of boost::optional from old versions of Boost, +// which exposed a dangling reference bug in Pybind11. Recent versions of +// boost::optional, as well as libstdc++'s std::optional, don't seem to be +// affected by the same issue. This is meant to be a minimal implementation +// required to reproduce the issue, not fully standard-compliant. +// See issue #3330 for more details. +template <typename T> +class ReferenceSensitiveOptional { +public: + using value_type = T; + + ReferenceSensitiveOptional() = default; + // NOLINTNEXTLINE(google-explicit-constructor) + ReferenceSensitiveOptional(const T &value) : storage{value} {} + // NOLINTNEXTLINE(google-explicit-constructor) + ReferenceSensitiveOptional(T &&value) : storage{std::move(value)} {} + ReferenceSensitiveOptional &operator=(const T &value) { + storage = {value}; + return *this; + } + ReferenceSensitiveOptional &operator=(T &&value) { + storage = {std::move(value)}; + return *this; + } + + template <typename... Args> + T &emplace(Args &&...args) { + storage.clear(); + storage.emplace_back(std::forward<Args>(args)...); + return storage.back(); + } + + const T &value() const noexcept { + assert(!storage.empty()); + return storage[0]; + } + + const T &operator*() const noexcept { return value(); } + + const T *operator->() const noexcept { return &value(); } + + explicit operator bool() const noexcept { return !storage.empty(); } + +private: + std::vector<T> storage; +}; + +namespace PYBIND11_NAMESPACE { +namespace detail { +template <typename T> +struct type_caster<ReferenceSensitiveOptional<T>> + : optional_caster<ReferenceSensitiveOptional<T>> {}; +} // namespace detail +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(stl, m) { // test_vector @@ -67,24 +173,29 @@ TEST_SUBMODULE(stl, m) { m.def("load_vector", [](const std::vector<int> &v) { return v.at(0) == 1 && v.at(1) == 2; }); // `std::vector<bool>` is special because it returns proxy objects instead of references m.def("cast_bool_vector", []() { return std::vector<bool>{true, false}; }); - m.def("load_bool_vector", [](const std::vector<bool> &v) { - return v.at(0) == true && v.at(1) == false; - }); + m.def("load_bool_vector", + [](const std::vector<bool> &v) { return v.at(0) == true && v.at(1) == false; }); // Unnumbered regression (caused by #936): pointers to stl containers aren't castable - static std::vector<RValueCaster> lvv{2}; - m.def("cast_ptr_vector", []() { return &lvv; }); + m.def( + "cast_ptr_vector", + []() { + // Using no-destructor idiom to side-step warnings from overzealous compilers. + static auto *v = new std::vector<RValueCaster>{2}; + return v; + }, + py::return_value_policy::reference); // test_deque m.def("cast_deque", []() { return std::deque<int>{1}; }); m.def("load_deque", [](const std::deque<int> &v) { return v.at(0) == 1 && v.at(1) == 2; }); // test_array - m.def("cast_array", []() { return std::array<int, 2> {{1 , 2}}; }); + m.def("cast_array", []() { return std::array<int, 2>{{1, 2}}; }); m.def("load_array", [](const std::array<int, 2> &a) { return a[0] == 1 && a[1] == 2; }); // test_valarray m.def("cast_valarray", []() { return std::valarray<int>{1, 4, 9}; }); - m.def("load_valarray", [](const std::valarray<int>& v) { + m.def("load_valarray", [](const std::valarray<int> &v) { return v.size() == 3 && v[0] == 1 && v[1] == 4 && v[2] == 9; }); @@ -97,7 +208,7 @@ TEST_SUBMODULE(stl, m) { // test_set m.def("cast_set", []() { return std::set<std::string>{"key1", "key2"}; }); m.def("load_set", [](const std::set<std::string> &set) { - return set.count("key1") && set.count("key2") && set.count("key3"); + return (set.count("key1") != 0u) && (set.count("key2") != 0u) && (set.count("key3") != 0u); }); // test_recursive_casting @@ -106,10 +217,12 @@ TEST_SUBMODULE(stl, m) { // NB: map and set keys are `const`, so while we technically do move them (as `const Type &&`), // casters don't typically do anything with that, which means they fall to the `const Type &` // caster. - m.def("cast_rv_map", []() { return std::unordered_map<std::string, RValueCaster>{{"a", RValueCaster{}}}; }); + m.def("cast_rv_map", []() { + return std::unordered_map<std::string, RValueCaster>{{"a", RValueCaster{}}}; + }); m.def("cast_rv_nested", []() { std::vector<std::array<std::list<std::unordered_map<std::string, RValueCaster>>, 2>> v; - v.emplace_back(); // add an array + v.emplace_back(); // add an array v.back()[0].emplace_back(); // add a map to the array v.back()[0].back().emplace("b", RValueCaster{}); v.back()[0].back().emplace("c", RValueCaster{}); @@ -118,15 +231,18 @@ TEST_SUBMODULE(stl, m) { return v; }); static std::array<RValueCaster, 2> lva; - static std::unordered_map<std::string, RValueCaster> lvm{{"a", RValueCaster{}}, {"b", RValueCaster{}}}; - static std::unordered_map<std::string, std::vector<std::list<std::array<RValueCaster, 2>>>> lvn; - lvn["a"].emplace_back(); // add a list + static std::unordered_map<std::string, RValueCaster> lvm{{"a", RValueCaster{}}, + {"b", RValueCaster{}}}; + static std::unordered_map<std::string, std::vector<std::list<std::array<RValueCaster, 2>>>> + lvn; + lvn["a"].emplace_back(); // add a list lvn["a"].back().emplace_back(); // add an array - lvn["a"].emplace_back(); // another list + lvn["a"].emplace_back(); // another list lvn["a"].back().emplace_back(); // add an array - lvn["b"].emplace_back(); // add a list + lvn["b"].emplace_back(); // add a list lvn["b"].back().emplace_back(); // add an array lvn["b"].back().emplace_back(); // add another array + static std::vector<RValueCaster> lvv{2}; m.def("cast_lv_vector", []() -> const decltype(lvv) & { return lvv; }); m.def("cast_lv_array", []() -> const decltype(lva) & { return lva; }); m.def("cast_lv_map", []() -> const decltype(lvm) & { return lvm; }); @@ -139,9 +255,15 @@ TEST_SUBMODULE(stl, m) { return v; }); + pybind11::enum_<EnumType>(m, "EnumType") + .value("kSet", EnumType::kSet) + .value("kUnset", EnumType::kUnset); + // test_move_out_container struct MoveOutContainer { - struct Value { int value; }; + struct Value { + int value; + }; std::list<Value> move_list() const { return {{0}, {1}, {2}}; } }; py::class_<MoveOutContainer::Value>(m, "MoveOutContainerValue") @@ -154,7 +276,7 @@ TEST_SUBMODULE(stl, m) { struct NoAssign { int value; - explicit NoAssign(int value = 0) : value(value) { } + explicit NoAssign(int value = 0) : value(value) {} NoAssign(const NoAssign &) = default; NoAssign(NoAssign &&) = default; @@ -165,13 +287,10 @@ TEST_SUBMODULE(stl, m) { .def(py::init<>()) .def(py::init<int>()); - - struct MoveOutDetector - { + struct MoveOutDetector { MoveOutDetector() = default; - MoveOutDetector(const MoveOutDetector&) = default; - MoveOutDetector(MoveOutDetector&& other) noexcept - : initialized(other.initialized) { + MoveOutDetector(const MoveOutDetector &) = default; + MoveOutDetector(MoveOutDetector &&other) noexcept : initialized(other.initialized) { // steal underlying resource other.initialized = false; } @@ -181,34 +300,37 @@ TEST_SUBMODULE(stl, m) { .def(py::init<>()) .def_readonly("initialized", &MoveOutDetector::initialized); - #ifdef PYBIND11_HAS_OPTIONAL // test_optional m.attr("has_optional") = true; using opt_int = std::optional<int>; using opt_no_assign = std::optional<NoAssign>; - m.def("double_or_zero", [](const opt_int& x) -> int { - return x.value_or(0) * 2; - }); - m.def("half_or_none", [](int x) -> opt_int { - return x ? opt_int(x / 2) : opt_int(); - }); - m.def("test_nullopt", [](opt_int x) { - return x.value_or(42); - }, py::arg_v("x", std::nullopt, "None")); - m.def("test_no_assign", [](const opt_no_assign &x) { - return x ? x->value : 42; - }, py::arg_v("x", std::nullopt, "None")); + m.def("double_or_zero", [](const opt_int &x) -> int { return x.value_or(0) * 2; }); + m.def("half_or_none", [](int x) -> opt_int { return x != 0 ? opt_int(x / 2) : opt_int(); }); + m.def( + "test_nullopt", + [](opt_int x) { return x.value_or(42); }, + py::arg_v("x", std::nullopt, "None")); + m.def( + "test_no_assign", + [](const opt_no_assign &x) { return x ? x->value : 42; }, + py::arg_v("x", std::nullopt, "None")); m.def("nodefer_none_optional", [](std::optional<int>) { return true; }); - m.def("nodefer_none_optional", [](py::none) { return false; }); + m.def("nodefer_none_optional", [](const py::none &) { return false; }); using opt_holder = OptionalHolder<std::optional, MoveOutDetector>; py::class_<opt_holder>(m, "OptionalHolder", "Class with optional member") .def(py::init<>()) .def_readonly("member", &opt_holder::member) .def("member_initialized", &opt_holder::member_initialized); + + using opt_props = OptionalProperties<std::optional>; + pybind11::class_<opt_props>(m, "OptionalProperties") + .def(pybind11::init<>()) + .def_property_readonly("access_by_ref", &opt_props::access_by_ref) + .def_property_readonly("access_by_copy", &opt_props::access_by_copy); #endif #ifdef PYBIND11_HAS_EXP_OPTIONAL @@ -217,27 +339,100 @@ TEST_SUBMODULE(stl, m) { using exp_opt_int = std::experimental::optional<int>; using exp_opt_no_assign = std::experimental::optional<NoAssign>; - m.def("double_or_zero_exp", [](const exp_opt_int& x) -> int { - return x.value_or(0) * 2; - }); - m.def("half_or_none_exp", [](int x) -> exp_opt_int { - return x ? exp_opt_int(x / 2) : exp_opt_int(); - }); - m.def("test_nullopt_exp", [](exp_opt_int x) { - return x.value_or(42); - }, py::arg_v("x", std::experimental::nullopt, "None")); - m.def("test_no_assign_exp", [](const exp_opt_no_assign &x) { - return x ? x->value : 42; - }, py::arg_v("x", std::experimental::nullopt, "None")); + m.def("double_or_zero_exp", [](const exp_opt_int &x) -> int { return x.value_or(0) * 2; }); + m.def("half_or_none_exp", + [](int x) -> exp_opt_int { return x ? exp_opt_int(x / 2) : exp_opt_int(); }); + m.def( + "test_nullopt_exp", + [](exp_opt_int x) { return x.value_or(42); }, + py::arg_v("x", std::experimental::nullopt, "None")); + m.def( + "test_no_assign_exp", + [](const exp_opt_no_assign &x) { return x ? x->value : 42; }, + py::arg_v("x", std::experimental::nullopt, "None")); using opt_exp_holder = OptionalHolder<std::experimental::optional, MoveOutDetector>; py::class_<opt_exp_holder>(m, "OptionalExpHolder", "Class with optional member") .def(py::init<>()) .def_readonly("member", &opt_exp_holder::member) .def("member_initialized", &opt_exp_holder::member_initialized); + + using opt_exp_props = OptionalProperties<std::experimental::optional>; + pybind11::class_<opt_exp_props>(m, "OptionalExpProperties") + .def(pybind11::init<>()) + .def_property_readonly("access_by_ref", &opt_exp_props::access_by_ref) + .def_property_readonly("access_by_copy", &opt_exp_props::access_by_copy); +#endif + +#if defined(PYBIND11_TEST_BOOST) + // test_boost_optional + m.attr("has_boost_optional") = true; + + using boost_opt_int = boost::optional<int>; + using boost_opt_no_assign = boost::optional<NoAssign>; + m.def("double_or_zero_boost", [](const boost_opt_int &x) -> int { return x.value_or(0) * 2; }); + m.def("half_or_none_boost", + [](int x) -> boost_opt_int { return x != 0 ? boost_opt_int(x / 2) : boost_opt_int(); }); + m.def( + "test_nullopt_boost", + [](boost_opt_int x) { return x.value_or(42); }, + py::arg_v("x", boost::none, "None")); + m.def( + "test_no_assign_boost", + [](const boost_opt_no_assign &x) { return x ? x->value : 42; }, + py::arg_v("x", boost::none, "None")); + + using opt_boost_holder = OptionalHolder<boost::optional, MoveOutDetector>; + py::class_<opt_boost_holder>(m, "OptionalBoostHolder", "Class with optional member") + .def(py::init<>()) + .def_readonly("member", &opt_boost_holder::member) + .def("member_initialized", &opt_boost_holder::member_initialized); + + using opt_boost_props = OptionalProperties<boost::optional>; + pybind11::class_<opt_boost_props>(m, "OptionalBoostProperties") + .def(pybind11::init<>()) + .def_property_readonly("access_by_ref", &opt_boost_props::access_by_ref) + .def_property_readonly("access_by_copy", &opt_boost_props::access_by_copy); +#endif + + // test_refsensitive_optional + using refsensitive_opt_int = ReferenceSensitiveOptional<int>; + using refsensitive_opt_no_assign = ReferenceSensitiveOptional<NoAssign>; + m.def("double_or_zero_refsensitive", + [](const refsensitive_opt_int &x) -> int { return (x ? x.value() : 0) * 2; }); + m.def("half_or_none_refsensitive", [](int x) -> refsensitive_opt_int { + return x != 0 ? refsensitive_opt_int(x / 2) : refsensitive_opt_int(); + }); + m.def( + "test_nullopt_refsensitive", + // NOLINTNEXTLINE(performance-unnecessary-value-param) + [](refsensitive_opt_int x) { return x ? x.value() : 42; }, + py::arg_v("x", refsensitive_opt_int(), "None")); + m.def( + "test_no_assign_refsensitive", + [](const refsensitive_opt_no_assign &x) { return x ? x->value : 42; }, + py::arg_v("x", refsensitive_opt_no_assign(), "None")); + + using opt_refsensitive_holder = OptionalHolder<ReferenceSensitiveOptional, MoveOutDetector>; + py::class_<opt_refsensitive_holder>( + m, "OptionalRefSensitiveHolder", "Class with optional member") + .def(py::init<>()) + .def_readonly("member", &opt_refsensitive_holder::member) + .def("member_initialized", &opt_refsensitive_holder::member_initialized); + + using opt_refsensitive_props = OptionalProperties<ReferenceSensitiveOptional>; + pybind11::class_<opt_refsensitive_props>(m, "OptionalRefSensitiveProperties") + .def(pybind11::init<>()) + .def_property_readonly("access_by_ref", &opt_refsensitive_props::access_by_ref) + .def_property_readonly("access_by_copy", &opt_refsensitive_props::access_by_copy); + +#ifdef PYBIND11_HAS_FILESYSTEM + // test_fs_path + m.attr("has_filesystem") = true; + m.def("parent_path", [](const std::filesystem::path &p) { return p.parent_path(); }); #endif -#ifdef PYBIND11_HAS_VARIANT +#ifdef PYBIND11_TEST_VARIANT static_assert(std::is_same<py::detail::variant_caster_visitor::result_type, py::handle>::value, "visitor::result_type is required by boost::variant in C++11 mode"); @@ -245,13 +440,16 @@ TEST_SUBMODULE(stl, m) { using result_type = const char *; result_type operator()(int) { return "int"; } - result_type operator()(std::string) { return "std::string"; } + result_type operator()(const std::string &) { return "std::string"; } result_type operator()(double) { return "double"; } result_type operator()(std::nullptr_t) { return "std::nullptr_t"; } +# if defined(PYBIND11_HAS_VARIANT) + result_type operator()(std::monostate) { return "std::monostate"; } +# endif }; // test_variant - m.def("load_variant", [](variant<int, std::string, double, std::nullptr_t> v) { + m.def("load_variant", [](const variant<int, std::string, double, std::nullptr_t> &v) { return py::detail::visit_helper<variant>::call(visitor(), v); }); m.def("load_variant_2pass", [](variant<double, int> v) { @@ -261,6 +459,18 @@ TEST_SUBMODULE(stl, m) { using V = variant<int, std::string>; return py::make_tuple(V(5), V("Hello")); }); + +# if defined(PYBIND11_HAS_VARIANT) + // std::monostate tests. + m.def("load_monostate_variant", + [](const variant<std::monostate, int, std::string> &v) -> const char * { + return py::detail::visit_helper<variant>::call(visitor(), v); + }); + m.def("cast_monostate_variant", []() { + using V = variant<std::monostate, int, std::string>; + return py::make_tuple(V{}, V(5), V("Hello")); + }); +# endif #endif // #528: templated constructor @@ -270,26 +480,32 @@ TEST_SUBMODULE(stl, m) { m.def("tpl_ctor_set", [](std::unordered_set<TplCtorClass> &) {}); #if defined(PYBIND11_HAS_OPTIONAL) m.def("tpl_constr_optional", [](std::optional<TplCtorClass> &) {}); -#elif defined(PYBIND11_HAS_EXP_OPTIONAL) - m.def("tpl_constr_optional", [](std::experimental::optional<TplCtorClass> &) {}); +#endif +#if defined(PYBIND11_HAS_EXP_OPTIONAL) + m.def("tpl_constr_optional_exp", [](std::experimental::optional<TplCtorClass> &) {}); +#endif +#if defined(PYBIND11_TEST_BOOST) + m.def("tpl_constr_optional_boost", [](boost::optional<TplCtorClass> &) {}); #endif // test_vec_of_reference_wrapper // #171: Can't return STL structures containing reference wrapper m.def("return_vec_of_reference_wrapper", [](std::reference_wrapper<UserType> p4) { static UserType p1{1}, p2{2}, p3{3}; - return std::vector<std::reference_wrapper<UserType>> { - std::ref(p1), std::ref(p2), std::ref(p3), p4 - }; + return std::vector<std::reference_wrapper<UserType>>{ + std::ref(p1), std::ref(p2), std::ref(p3), p4}; }); // test_stl_pass_by_pointer - m.def("stl_pass_by_pointer", [](std::vector<int>* v) { return *v; }, "v"_a=nullptr); + m.def( + "stl_pass_by_pointer", [](std::vector<int> *v) { return *v; }, "v"_a = nullptr); // #1258: pybind11/stl.h converts string to vector<string> - m.def("func_with_string_or_vector_string_arg_overload", [](std::vector<std::string>) { return 1; }); - m.def("func_with_string_or_vector_string_arg_overload", [](std::list<std::string>) { return 2; }); - m.def("func_with_string_or_vector_string_arg_overload", [](std::string) { return 3; }); + m.def("func_with_string_or_vector_string_arg_overload", + [](const std::vector<std::string> &) { return 1; }); + m.def("func_with_string_or_vector_string_arg_overload", + [](const std::list<std::string> &) { return 2; }); + m.def("func_with_string_or_vector_string_arg_overload", [](const std::string &) { return 3; }); class Placeholder { public: @@ -300,19 +516,24 @@ TEST_SUBMODULE(stl, m) { py::class_<Placeholder>(m, "Placeholder"); /// test_stl_vector_ownership - m.def("test_stl_ownership", - []() { - std::vector<Placeholder *> result; - result.push_back(new Placeholder()); - return result; - }, - py::return_value_policy::take_ownership); + m.def( + "test_stl_ownership", + []() { + std::vector<Placeholder *> result; + result.push_back(new Placeholder()); + return result; + }, + py::return_value_policy::take_ownership); m.def("array_cast_sequence", [](std::array<int, 3> x) { return x; }); /// test_issue_1561 - struct Issue1561Inner { std::string data; }; - struct Issue1561Outer { std::vector<Issue1561Inner> list; }; + struct Issue1561Inner { + std::string data; + }; + struct Issue1561Outer { + std::vector<Issue1561Inner> list; + }; py::class_<Issue1561Inner>(m, "Issue1561Inner") .def(py::init<std::string>()) @@ -321,4 +542,10 @@ TEST_SUBMODULE(stl, m) { py::class_<Issue1561Outer>(m, "Issue1561Outer") .def(py::init<>()) .def_readwrite("list", &Issue1561Outer::list); + + m.def( + "return_vector_bool_raw_ptr", + []() { return new std::vector<bool>(4513); }, + // Without explicitly specifying `take_ownership`, this function leaks. + py::return_value_policy::take_ownership); } diff --git a/tests/test_stl.py b/tests/test_stl.py index 33001754..8a614f8b 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- import pytest +from pybind11_tests import ConstructorStats, UserType from pybind11_tests import stl as m -from pybind11_tests import UserType -from pybind11_tests import ConstructorStats def test_vector(doc): @@ -16,6 +14,7 @@ def test_vector(doc): assert m.cast_bool_vector() == [True, False] assert m.load_bool_vector([True, False]) + assert m.load_bool_vector((True, False)) assert doc(m.cast_vector) == "cast_vector() -> List[int]" assert doc(m.load_vector) == "load_vector(arg0: List[int]) -> bool" @@ -24,7 +23,7 @@ def test_vector(doc): assert m.cast_ptr_vector() == ["lvalue", "lvalue"] -def test_deque(doc): +def test_deque(): """std::deque <-> list""" lst = m.cast_deque() assert lst == [1] @@ -38,9 +37,13 @@ def test_array(doc): lst = m.cast_array() assert lst == [1, 2] assert m.load_array(lst) + assert m.load_array(tuple(lst)) - assert doc(m.cast_array) == "cast_array() -> List[int[2]]" - assert doc(m.load_array) == "load_array(arg0: List[int[2]]) -> bool" + assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]" + assert ( + doc(m.load_array) + == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool" + ) def test_valarray(doc): @@ -48,6 +51,7 @@ def test_valarray(doc): lst = m.cast_valarray() assert lst == [1, 4, 9] assert m.load_valarray(lst) + assert m.load_valarray(tuple(lst)) assert doc(m.cast_valarray) == "cast_valarray() -> List[int]" assert doc(m.load_valarray) == "load_valarray(arg0: List[int]) -> bool" @@ -72,6 +76,7 @@ def test_set(doc): assert s == {"key1", "key2"} s.add("key3") assert m.load_set(s) + assert m.load_set(frozenset(s)) assert doc(m.cast_set) == "cast_set() -> Set[str]" assert doc(m.load_set) == "load_set(arg0: Set[str]) -> bool" @@ -93,7 +98,8 @@ def test_recursive_casting(): # Issue #853 test case: z = m.cast_unique_ptr_vector() - assert z[0].value == 7 and z[1].value == 42 + assert z[0].value == 7 + assert z[1].value == 42 def test_move_out_container(): @@ -133,6 +139,10 @@ def test_optional(): assert mvalue.initialized assert holder.member_initialized() + props = m.OptionalProperties() + assert int(props.access_by_ref) == 42 + assert int(props.access_by_copy) == 42 + @pytest.mark.skipif( not hasattr(m, "has_exp_optional"), reason="no <experimental/optional>" @@ -161,6 +171,88 @@ def test_exp_optional(): assert mvalue.initialized assert holder.member_initialized() + props = m.OptionalExpProperties() + assert int(props.access_by_ref) == 42 + assert int(props.access_by_copy) == 42 + + +@pytest.mark.skipif(not hasattr(m, "has_boost_optional"), reason="no <boost/optional>") +def test_boost_optional(): + assert m.double_or_zero_boost(None) == 0 + assert m.double_or_zero_boost(42) == 84 + pytest.raises(TypeError, m.double_or_zero_boost, "foo") + + assert m.half_or_none_boost(0) is None + assert m.half_or_none_boost(42) == 21 + pytest.raises(TypeError, m.half_or_none_boost, "foo") + + assert m.test_nullopt_boost() == 42 + assert m.test_nullopt_boost(None) == 42 + assert m.test_nullopt_boost(42) == 42 + assert m.test_nullopt_boost(43) == 43 + + assert m.test_no_assign_boost() == 42 + assert m.test_no_assign_boost(None) == 42 + assert m.test_no_assign_boost(m.NoAssign(43)) == 43 + pytest.raises(TypeError, m.test_no_assign_boost, 43) + + holder = m.OptionalBoostHolder() + mvalue = holder.member + assert mvalue.initialized + assert holder.member_initialized() + + props = m.OptionalBoostProperties() + assert int(props.access_by_ref) == 42 + assert int(props.access_by_copy) == 42 + + +def test_reference_sensitive_optional(): + assert m.double_or_zero_refsensitive(None) == 0 + assert m.double_or_zero_refsensitive(42) == 84 + pytest.raises(TypeError, m.double_or_zero_refsensitive, "foo") + + assert m.half_or_none_refsensitive(0) is None + assert m.half_or_none_refsensitive(42) == 21 + pytest.raises(TypeError, m.half_or_none_refsensitive, "foo") + + assert m.test_nullopt_refsensitive() == 42 + assert m.test_nullopt_refsensitive(None) == 42 + assert m.test_nullopt_refsensitive(42) == 42 + assert m.test_nullopt_refsensitive(43) == 43 + + assert m.test_no_assign_refsensitive() == 42 + assert m.test_no_assign_refsensitive(None) == 42 + assert m.test_no_assign_refsensitive(m.NoAssign(43)) == 43 + pytest.raises(TypeError, m.test_no_assign_refsensitive, 43) + + holder = m.OptionalRefSensitiveHolder() + mvalue = holder.member + assert mvalue.initialized + assert holder.member_initialized() + + props = m.OptionalRefSensitiveProperties() + assert int(props.access_by_ref) == 42 + assert int(props.access_by_copy) == 42 + + +@pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no <filesystem>") +def test_fs_path(): + from pathlib import Path + + class PseudoStrPath: + def __fspath__(self): + return "foo/bar" + + class PseudoBytesPath: + def __fspath__(self): + return b"foo/bar" + + assert m.parent_path(Path("foo/bar")) == Path("foo") + assert m.parent_path("foo/bar") == Path("foo") + assert m.parent_path(b"foo/bar") == Path("foo") + assert m.parent_path(PseudoStrPath()) == Path("foo") + assert m.parent_path(PseudoBytesPath()) == Path("foo") + @pytest.mark.skipif(not hasattr(m, "load_variant"), reason="no <variant>") def test_variant(doc): @@ -179,6 +271,22 @@ def test_variant(doc): ) +@pytest.mark.skipif( + not hasattr(m, "load_monostate_variant"), reason="no std::monostate" +) +def test_variant_monostate(doc): + assert m.load_monostate_variant(None) == "std::monostate" + assert m.load_monostate_variant(1) == "int" + assert m.load_monostate_variant("1") == "std::string" + + assert m.cast_monostate_variant() == (None, 5, "Hello") + + assert ( + doc(m.load_monostate_variant) + == "load_monostate_variant(arg0: Union[None, int, str]) -> str" + ) + + def test_vec_of_reference_wrapper(): """#171: Can't return reference wrappers (or STL structures containing them)""" assert ( @@ -198,7 +306,7 @@ def test_stl_pass_by_pointer(msg): 1. (v: List[int] = None) -> List[int] Invoked with: - """ # noqa: E501 line too long + """ ) with pytest.raises(TypeError) as excinfo: @@ -210,7 +318,7 @@ def test_stl_pass_by_pointer(msg): 1. (v: List[int] = None) -> List[int] Invoked with: None - """ # noqa: E501 line too long + """ ) assert m.stl_pass_by_pointer([1, 2, 3]) == [1, 2, 3] @@ -259,8 +367,15 @@ def test_array_cast_sequence(): def test_issue_1561(): - """ check fix for issue #1561 """ + """check fix for issue #1561""" bar = m.Issue1561Outer() bar.list = [m.Issue1561Inner("bar")] - bar.list + assert bar.list assert bar.list[0].data == "bar" + + +def test_return_vector_bool_raw_ptr(): + # Add `while True:` for manual leak checking. + v = m.return_vector_bool_raw_ptr() + assert isinstance(v, list) + assert len(v) == 4513 diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index c791477e..1681760a 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -7,23 +7,24 @@ BSD-style license that can be found in the LICENSE file. */ +#include <pybind11/numpy.h> +#include <pybind11/stl_bind.h> + #include "pybind11_tests.h" -#include <pybind11/stl_bind.h> -#include <pybind11/numpy.h> -#include <map> #include <deque> +#include <map> #include <unordered_map> class El { public: El() = delete; - El(int v) : a(v) { } + explicit El(int v) : a(v) {} int a; }; -std::ostream & operator<<(std::ostream &s, El const&v) { +std::ostream &operator<<(std::ostream &s, El const &v) { s << "El{" << v.a << '}'; return s; } @@ -40,35 +41,79 @@ public: int value; }; -template <class Container> Container *one_to_n(int n) { - auto v = new Container(); - for (int i = 1; i <= n; i++) +template <class Container> +Container *one_to_n(int n) { + auto *v = new Container(); + for (int i = 1; i <= n; i++) { v->emplace_back(i); + } return v; } -template <class Map> Map *times_ten(int n) { - auto m = new Map(); - for (int i = 1; i <= n; i++) - m->emplace(int(i), E_nc(10*i)); +template <class Map> +Map *times_ten(int n) { + auto *m = new Map(); + for (int i = 1; i <= n; i++) { + m->emplace(int(i), E_nc(10 * i)); + } return m; } -template <class NestMap> NestMap *times_hundred(int n) { - auto m = new NestMap(); - for (int i = 1; i <= n; i++) - for (int j = 1; j <= n; j++) - (*m)[i].emplace(int(j*10), E_nc(100*j)); +template <class NestMap> +NestMap *times_hundred(int n) { + auto *m = new NestMap(); + for (int i = 1; i <= n; i++) { + for (int j = 1; j <= n; j++) { + (*m)[i].emplace(int(j * 10), E_nc(100 * j)); + } + } return m; } +/* + * Recursive data structures as test for issue #4623 + */ +struct RecursiveVector : std::vector<RecursiveVector> { + using Parent = std::vector<RecursiveVector>; + using Parent::Parent; +}; + +struct RecursiveMap : std::map<int, RecursiveMap> { + using Parent = std::map<int, RecursiveMap>; + using Parent::Parent; +}; + +/* + * Pybind11 does not catch more complicated recursion schemes, such as mutual + * recursion. + * In that case custom recursive_container_traits specializations need to be added, + * thus manually telling pybind11 about the recursion. + */ +struct MutuallyRecursiveContainerPairMV; +struct MutuallyRecursiveContainerPairVM; + +struct MutuallyRecursiveContainerPairMV : std::map<int, MutuallyRecursiveContainerPairVM> {}; +struct MutuallyRecursiveContainerPairVM : std::vector<MutuallyRecursiveContainerPairMV> {}; + +namespace pybind11 { +namespace detail { +template <typename SFINAE> +struct recursive_container_traits<MutuallyRecursiveContainerPairMV, SFINAE> { + using type_to_check_recursively = recursive_bottom; +}; +template <typename SFINAE> +struct recursive_container_traits<MutuallyRecursiveContainerPairVM, SFINAE> { + using type_to_check_recursively = recursive_bottom; +}; +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(stl_binders, m) { // test_vector_int py::bind_vector<std::vector<unsigned int>>(m, "VectorInt", py::buffer_protocol()); // test_vector_custom - py::class_<El>(m, "El") - .def(py::init<int>()); + py::class_<El>(m, "El").def(py::init<int>()); py::bind_vector<std::vector<El>>(m, "VectorEl"); py::bind_vector<std::vector<std::vector<El>>>(m, "VectorVectorEl"); @@ -78,11 +123,10 @@ TEST_SUBMODULE(stl_binders, m) { // test_map_string_double_const py::bind_map<std::map<std::string, double const>>(m, "MapStringDoubleConst"); - py::bind_map<std::unordered_map<std::string, double const>>(m, "UnorderedMapStringDoubleConst"); + py::bind_map<std::unordered_map<std::string, double const>>(m, + "UnorderedMapStringDoubleConst"); - py::class_<E_nc>(m, "ENC") - .def(py::init<int>()) - .def_readwrite("value", &E_nc::value); + py::class_<E_nc>(m, "ENC").def(py::init<int>()).def_readwrite("value", &E_nc::value); // test_noncopyable_containers py::bind_vector<std::vector<E_nc>>(m, "VectorENC"); @@ -95,14 +139,15 @@ TEST_SUBMODULE(stl_binders, m) { m.def("get_umnc", ×_ten<std::unordered_map<int, E_nc>>); // Issue #1885: binding nested std::map<X, Container<E>> with E non-copyable py::bind_map<std::map<int, std::vector<E_nc>>>(m, "MapVecENC"); - m.def("get_nvnc", [](int n) - { - auto m = new std::map<int, std::vector<E_nc>>(); - for (int i = 1; i <= n; i++) - for (int j = 1; j <= n; j++) - (*m)[i].emplace_back(j); - return m; - }); + m.def("get_nvnc", [](int n) { + auto *m = new std::map<int, std::vector<E_nc>>(); + for (int i = 1; i <= n; i++) { + for (int j = 1; j <= n; j++) { + (*m)[i].emplace_back(j); + } + } + return m; + }); py::bind_map<std::map<int, std::map<int, E_nc>>>(m, "MapMapENC"); m.def("get_nmnc", ×_hundred<std::map<int, std::map<int, E_nc>>>); py::bind_map<std::unordered_map<int, std::unordered_map<int, E_nc>>>(m, "UmapUmapENC"); @@ -111,19 +156,41 @@ TEST_SUBMODULE(stl_binders, m) { // test_vector_buffer py::bind_vector<std::vector<unsigned char>>(m, "VectorUChar", py::buffer_protocol()); // no dtype declared for this version: - struct VUndeclStruct { bool w; uint32_t x; double y; bool z; }; - m.def("create_undeclstruct", [m] () mutable { - py::bind_vector<std::vector<VUndeclStruct>>(m, "VectorUndeclStruct", py::buffer_protocol()); + struct VUndeclStruct { + bool w; + uint32_t x; + double y; + bool z; + }; + m.def("create_undeclstruct", [m]() mutable { + py::bind_vector<std::vector<VUndeclStruct>>( + m, "VectorUndeclStruct", py::buffer_protocol()); }); + // Bind recursive container types + py::bind_vector<RecursiveVector>(m, "RecursiveVector"); + py::bind_map<RecursiveMap>(m, "RecursiveMap"); + py::bind_map<MutuallyRecursiveContainerPairMV>(m, "MutuallyRecursiveContainerPairMV"); + py::bind_vector<MutuallyRecursiveContainerPairVM>(m, "MutuallyRecursiveContainerPairVM"); + // The rest depends on numpy: - try { py::module_::import("numpy"); } - catch (...) { return; } + try { + py::module_::import("numpy"); + } catch (...) { + return; + } // test_vector_buffer_numpy - struct VStruct { bool w; uint32_t x; double y; bool z; }; + struct VStruct { + bool w; + uint32_t x; + double y; + bool z; + }; PYBIND11_NUMPY_DTYPE(VStruct, w, x, y, z); py::class_<VStruct>(m, "VStruct").def_readwrite("x", &VStruct::x); py::bind_vector<std::vector<VStruct>>(m, "VectorStruct", py::buffer_protocol()); - m.def("get_vectorstruct", [] {return std::vector<VStruct> {{0, 5, 3.0, 1}, {1, 30, -1e4, 0}};}); + m.def("get_vectorstruct", [] { + return std::vector<VStruct>{{false, 5, 3.0, true}, {true, 30, -1e4, false}}; + }); } diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 84132a2b..e002f5b6 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -1,8 +1,5 @@ -# -*- coding: utf-8 -*- import pytest -import env # noqa: F401 - from pybind11_tests import stl_binders as m @@ -75,18 +72,13 @@ def test_vector_buffer(): assert v[1] == 2 v[2] = 5 mv = memoryview(v) # We expose the buffer interface - if not env.PY2: - assert mv[2] == 5 - mv[2] = 6 - else: - assert mv[2] == "\x05" - mv[2] = "\x06" + assert mv[2] == 5 + mv[2] = 6 assert v[2] == 6 - if not env.PY2: - mv = memoryview(b) - v = m.VectorUChar(mv[::2]) - assert v[1] == 3 + mv = memoryview(b) + v = m.VectorUChar(mv[::2]) + assert v[1] == 3 with pytest.raises(RuntimeError) as excinfo: m.create_undeclstruct() # Undeclared struct contents, no buffer interface @@ -161,15 +153,43 @@ def test_map_string_double(): mm["b"] = 2.5 assert list(mm) == ["a", "b"] - assert list(mm.items()) == [("a", 1), ("b", 2.5)] assert str(mm) == "MapStringDouble{a: 1, b: 2.5}" + assert "b" in mm + assert "c" not in mm + assert 123 not in mm + + # Check that keys, values, items are views, not merely iterable + keys = mm.keys() + values = mm.values() + items = mm.items() + assert list(keys) == ["a", "b"] + assert len(keys) == 2 + assert "a" in keys + assert "c" not in keys + assert 123 not in keys + assert list(items) == [("a", 1), ("b", 2.5)] + assert len(items) == 2 + assert ("b", 2.5) in items + assert "hello" not in items + assert ("b", 2.5, None) not in items + assert list(values) == [1, 2.5] + assert len(values) == 2 + assert 1 in values + assert 2 not in values + # Check that views update when the map is updated + mm["c"] = -1 + assert list(keys) == ["a", "b", "c"] + assert list(values) == [1, 2.5, -1] + assert list(items) == [("a", 1), ("b", 2.5), ("c", -1)] um = m.UnorderedMapStringDouble() um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um) == ["ua", "ub"] + assert list(um.keys()) == list(um) + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] + assert list(zip(um.keys(), um.values())) == list(um.items()) assert "UnorderedMapStringDouble" in str(um) @@ -284,8 +304,52 @@ def test_map_delitem(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um) == ["ua", "ub"] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] del um["ua"] - assert sorted(list(um)) == ["ub"] - assert sorted(list(um.items())) == [("ub", 2.6)] + assert sorted(um) == ["ub"] + assert sorted(um.items()) == [("ub", 2.6)] + + +def test_map_view_types(): + map_string_double = m.MapStringDouble() + unordered_map_string_double = m.UnorderedMapStringDouble() + map_string_double_const = m.MapStringDoubleConst() + unordered_map_string_double_const = m.UnorderedMapStringDoubleConst() + + assert map_string_double.keys().__class__.__name__ == "KeysView[str]" + assert map_string_double.values().__class__.__name__ == "ValuesView[float]" + assert map_string_double.items().__class__.__name__ == "ItemsView[str, float]" + + keys_type = type(map_string_double.keys()) + assert type(unordered_map_string_double.keys()) is keys_type + assert type(map_string_double_const.keys()) is keys_type + assert type(unordered_map_string_double_const.keys()) is keys_type + + values_type = type(map_string_double.values()) + assert type(unordered_map_string_double.values()) is values_type + assert type(map_string_double_const.values()) is values_type + assert type(unordered_map_string_double_const.values()) is values_type + + items_type = type(map_string_double.items()) + assert type(unordered_map_string_double.items()) is items_type + assert type(map_string_double_const.items()) is items_type + assert type(unordered_map_string_double_const.items()) is items_type + + +def test_recursive_vector(): + recursive_vector = m.RecursiveVector() + recursive_vector.append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + # Can't use len() since test_stl_binders.cpp does not include stl.h, + # so the necessary conversion is missing + assert recursive_vector[0].count(m.RecursiveVector()) == 2 + + +def test_recursive_map(): + recursive_map = m.RecursiveMap() + recursive_map[100] = m.RecursiveMap() + recursive_map[100][101] = m.RecursiveMap() + recursive_map[100][102] = m.RecursiveMap() + assert list(recursive_map[100].keys()) == [101, 102] diff --git a/tests/test_tagbased_polymorphic.cpp b/tests/test_tagbased_polymorphic.cpp index 838a168d..12ba6532 100644 --- a/tests/test_tagbased_polymorphic.cpp +++ b/tests/test_tagbased_polymorphic.cpp @@ -7,11 +7,11 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" #include <pybind11/stl.h> -struct Animal -{ +#include "pybind11_tests.h" + +struct Animal { // Make this type also a "standard" polymorphic type, to confirm that // specializing polymorphic_type_hook using enable_if_t still works // (https://github.com/pybind/pybind11/pull/2016/). @@ -20,55 +20,54 @@ struct Animal // Enum for tag-based polymorphism. enum class Kind { Unknown = 0, - Dog = 100, Labrador, Chihuahua, LastDog = 199, - Cat = 200, Panther, LastCat = 299 + Dog = 100, + Labrador, + Chihuahua, + LastDog = 199, + Cat = 200, + Panther, + LastCat = 299 }; - static const std::type_info* type_of_kind(Kind kind); + static const std::type_info *type_of_kind(Kind kind); static std::string name_of_kind(Kind kind); const Kind kind; const std::string name; - protected: - Animal(const std::string& _name, Kind _kind) - : kind(_kind), name(_name) - {} +protected: + Animal(const std::string &_name, Kind _kind) : kind(_kind), name(_name) {} }; -struct Dog : Animal -{ - Dog(const std::string& _name, Kind _kind = Kind::Dog) : Animal(_name, _kind) {} +struct Dog : Animal { + explicit Dog(const std::string &_name, Kind _kind = Kind::Dog) : Animal(_name, _kind) {} std::string bark() const { return name_of_kind(kind) + " " + name + " goes " + sound; } std::string sound = "WOOF!"; }; -struct Labrador : Dog -{ - Labrador(const std::string& _name, int _excitement = 9001) +struct Labrador : Dog { + explicit Labrador(const std::string &_name, int _excitement = 9001) : Dog(_name, Kind::Labrador), excitement(_excitement) {} int excitement; }; -struct Chihuahua : Dog -{ - Chihuahua(const std::string& _name) : Dog(_name, Kind::Chihuahua) { sound = "iyiyiyiyiyi"; } +struct Chihuahua : Dog { + explicit Chihuahua(const std::string &_name) : Dog(_name, Kind::Chihuahua) { + sound = "iyiyiyiyiyi"; + } std::string bark() const { return Dog::bark() + " and runs in circles"; } }; -struct Cat : Animal -{ - Cat(const std::string& _name, Kind _kind = Kind::Cat) : Animal(_name, _kind) {} +struct Cat : Animal { + explicit Cat(const std::string &_name, Kind _kind = Kind::Cat) : Animal(_name, _kind) {} std::string purr() const { return "mrowr"; } }; -struct Panther : Cat -{ - Panther(const std::string& _name) : Cat(_name, Kind::Panther) {} +struct Panther : Cat { + explicit Panther(const std::string &_name) : Cat(_name, Kind::Panther) {} std::string purr() const { return "mrrrRRRRRR"; } }; -std::vector<std::unique_ptr<Animal>> create_zoo() -{ +std::vector<std::unique_ptr<Animal>> create_zoo() { std::vector<std::unique_ptr<Animal>> ret; ret.emplace_back(new Labrador("Fido", 15000)); @@ -83,45 +82,53 @@ std::vector<std::unique_ptr<Animal>> create_zoo() return ret; } -const std::type_info* Animal::type_of_kind(Kind kind) -{ +const std::type_info *Animal::type_of_kind(Kind kind) { switch (kind) { - case Kind::Unknown: break; - - case Kind::Dog: break; - case Kind::Labrador: return &typeid(Labrador); - case Kind::Chihuahua: return &typeid(Chihuahua); - case Kind::LastDog: break; - - case Kind::Cat: break; - case Kind::Panther: return &typeid(Panther); - case Kind::LastCat: break; + case Kind::Unknown: + case Kind::Dog: + break; + + case Kind::Labrador: + return &typeid(Labrador); + case Kind::Chihuahua: + return &typeid(Chihuahua); + + case Kind::LastDog: + case Kind::Cat: + break; + case Kind::Panther: + return &typeid(Panther); + case Kind::LastCat: + break; } - if (kind >= Kind::Dog && kind <= Kind::LastDog) return &typeid(Dog); - if (kind >= Kind::Cat && kind <= Kind::LastCat) return &typeid(Cat); + if (kind >= Kind::Dog && kind <= Kind::LastDog) { + return &typeid(Dog); + } + if (kind >= Kind::Cat && kind <= Kind::LastCat) { + return &typeid(Cat); + } return nullptr; } -std::string Animal::name_of_kind(Kind kind) -{ +std::string Animal::name_of_kind(Kind kind) { std::string raw_name = type_of_kind(kind)->name(); py::detail::clean_type_id(raw_name); return raw_name; } -namespace pybind11 { - template <typename itype> - struct polymorphic_type_hook<itype, detail::enable_if_t<std::is_base_of<Animal, itype>::value>> - { - static const void *get(const itype *src, const std::type_info*& type) - { type = src ? Animal::type_of_kind(src->kind) : nullptr; return src; } - }; -} // namespace pybind11 +namespace PYBIND11_NAMESPACE { +template <typename itype> +struct polymorphic_type_hook<itype, detail::enable_if_t<std::is_base_of<Animal, itype>::value>> { + static const void *get(const itype *src, const std::type_info *&type) { + type = src ? Animal::type_of_kind(src->kind) : nullptr; + return src; + } +}; +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(tagbased_polymorphic, m) { - py::class_<Animal>(m, "Animal") - .def_readonly("name", &Animal::name); + py::class_<Animal>(m, "Animal").def_readonly("name", &Animal::name); py::class_<Dog, Animal>(m, "Dog") .def(py::init<std::string>()) .def_readwrite("sound", &Dog::sound) @@ -132,9 +139,7 @@ TEST_SUBMODULE(tagbased_polymorphic, m) { py::class_<Chihuahua, Dog>(m, "Chihuahua") .def(py::init<std::string>()) .def("bark", &Chihuahua::bark); - py::class_<Cat, Animal>(m, "Cat") - .def(py::init<std::string>()) - .def("purr", &Cat::purr); + py::class_<Cat, Animal>(m, "Cat").def(py::init<std::string>()).def("purr", &Cat::purr); py::class_<Panther, Cat>(m, "Panther") .def(py::init<std::string>()) .def("purr", &Panther::purr); diff --git a/tests/test_tagbased_polymorphic.py b/tests/test_tagbased_polymorphic.py index 64eb8a3c..84f0ea71 100644 --- a/tests/test_tagbased_polymorphic.py +++ b/tests/test_tagbased_polymorphic.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from pybind11_tests import tagbased_polymorphic as m diff --git a/tests/test_thread.cpp b/tests/test_thread.cpp new file mode 100644 index 00000000..1536503b --- /dev/null +++ b/tests/test_thread.cpp @@ -0,0 +1,66 @@ +/* + tests/test_thread.cpp -- call pybind11 bound methods in threads + + Copyright (c) 2021 Laramie Leavitt (Google LLC) <lar@google.com> + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include <pybind11/cast.h> +#include <pybind11/pybind11.h> + +#include "pybind11_tests.h" + +#include <chrono> +#include <thread> + +namespace py = pybind11; + +namespace { + +struct IntStruct { + explicit IntStruct(int v) : value(v){}; + ~IntStruct() { value = -value; } + IntStruct(const IntStruct &) = default; + IntStruct &operator=(const IntStruct &) = default; + + int value; +}; + +} // namespace + +TEST_SUBMODULE(thread, m) { + + py::class_<IntStruct>(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); })); + + // implicitly_convertible uses loader_life_support when an implicit + // conversion is required in order to lifetime extend the reference. + // + // This test should be run with ASAN for better effectiveness. + py::implicitly_convertible<int, IntStruct>(); + + m.def("test", [](int expected, const IntStruct &in) { + { + py::gil_scoped_release release; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + + if (in.value != expected) { + throw std::runtime_error("Value changed!!"); + } + }); + + m.def( + "test_no_gil", + [](int expected, const IntStruct &in) { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + if (in.value != expected) { + throw std::runtime_error("Value changed!!"); + } + }, + py::call_guard<py::gil_scoped_release>()); + + // NOTE: std::string_view also uses loader_life_support to ensure that + // the string contents remain alive, but that's a C++ 17 feature. +} diff --git a/tests/test_thread.py b/tests/test_thread.py new file mode 100644 index 00000000..e89991f9 --- /dev/null +++ b/tests/test_thread.py @@ -0,0 +1,42 @@ +import threading + +from pybind11_tests import thread as m + + +class Thread(threading.Thread): + def __init__(self, fn): + super().__init__() + self.fn = fn + self.e = None + + def run(self): + try: + for i in range(10): + self.fn(i, i) + except Exception as e: + self.e = e + + def join(self): + super().join() + if self.e: + raise self.e + + +def test_implicit_conversion(): + a = Thread(m.test) + b = Thread(m.test) + c = Thread(m.test) + for x in [a, b, c]: + x.start() + for x in [c, b, a]: + x.join() + + +def test_implicit_conversion_no_gil(): + a = Thread(m.test_no_gil) + b = Thread(m.test_no_gil) + c = Thread(m.test_no_gil) + for x in [a, b, c]: + x.start() + for x in [c, b, a]: + x.join() diff --git a/tests/test_type_caster_pyobject_ptr.cpp b/tests/test_type_caster_pyobject_ptr.cpp new file mode 100644 index 00000000..1667ea12 --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.cpp @@ -0,0 +1,130 @@ +#include <pybind11/functional.h> +#include <pybind11/stl.h> +#include <pybind11/type_caster_pyobject_ptr.h> + +#include "pybind11_tests.h" + +#include <cstddef> +#include <vector> + +namespace { + +std::vector<PyObject *> make_vector_pyobject_ptr(const py::object &ValueHolder) { + std::vector<PyObject *> vec_obj; + for (int i = 1; i < 3; i++) { + vec_obj.push_back(ValueHolder(i * 93).release().ptr()); + } + // This vector now owns the refcounts. + return vec_obj; +} + +} // namespace + +TEST_SUBMODULE(type_caster_pyobject_ptr, m) { + m.def("cast_from_pyobject_ptr", []() { + PyObject *ptr = PyLong_FromLongLong(6758L); + return py::cast(ptr, py::return_value_policy::take_ownership); + }); + m.def("cast_handle_to_pyobject_ptr", [](py::handle obj) { + auto rc1 = obj.ref_count(); + auto *ptr = py::cast<PyObject *>(obj); + auto rc2 = obj.ref_count(); + if (rc2 != rc1 + 1) { + return -1; + } + return 100 - py::reinterpret_steal<py::object>(ptr).attr("value").cast<int>(); + }); + m.def("cast_object_to_pyobject_ptr", [](py::object obj) { + py::handle hdl = obj; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast<PyObject *>(std::move(obj)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 300 - py::reinterpret_steal<py::object>(ptr).attr("value").cast<int>(); + }); + m.def("cast_list_to_pyobject_ptr", [](py::list lst) { + // This is to cover types implicitly convertible to object. + py::handle hdl = lst; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast<PyObject *>(std::move(lst)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 400 - static_cast<int>(py::len(py::reinterpret_steal<py::list>(ptr))); + }); + + m.def( + "return_pyobject_ptr", + []() { return PyLong_FromLongLong(2314L); }, + py::return_value_policy::take_ownership); + m.def("pass_pyobject_ptr", [](PyObject *ptr) { + return 200 - py::reinterpret_borrow<py::object>(ptr).attr("value").cast<int>(); + }); + + m.def("call_callback_with_object_return", + [](const std::function<py::object(int)> &cb, int value) { return cb(value); }); + m.def( + "call_callback_with_pyobject_ptr_return", + [](const std::function<PyObject *(int)> &cb, int value) { return cb(value); }, + py::return_value_policy::take_ownership); + m.def( + "call_callback_with_pyobject_ptr_arg", + [](const std::function<int(PyObject *)> &cb, py::handle obj) { return cb(obj.ptr()); }, + py::arg("cb"), // This triggers return_value_policy::automatic_reference + py::arg("obj")); + + m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) { + if (set_error) { + PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling."); + } + PyObject *ptr = nullptr; + py::cast(ptr); + }); + + m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() { + PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling."); + py::cast(Py_None); + }); + + m.def("pass_list_pyobject_ptr", [](const std::vector<PyObject *> &vec_obj) { + int acc = 0; + for (const auto &ptr : vec_obj) { + acc = acc * 1000 + py::reinterpret_borrow<py::object>(ptr).attr("value").cast<int>(); + } + return acc; + }); + + m.def("return_list_pyobject_ptr_take_ownership", + make_vector_pyobject_ptr, + // Ownership is transferred one-by-one when the vector is converted to a Python list. + py::return_value_policy::take_ownership); + + m.def("return_list_pyobject_ptr_reference", + make_vector_pyobject_ptr, + // Ownership is not transferred. + py::return_value_policy::reference); + + m.def("dec_ref_each_pyobject_ptr", [](const std::vector<PyObject *> &vec_obj) { + std::size_t i = 0; + for (; i < vec_obj.size(); i++) { + py::handle h(vec_obj[i]); + if (static_cast<std::size_t>(h.ref_count()) < 2) { + break; // Something is badly wrong. + } + h.dec_ref(); + } + return i; + }); + + m.def("pass_pyobject_ptr_and_int", [](PyObject *, int) {}); + +#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing. + { + PyObject *ptr = nullptr; + (void) py::cast(*ptr); + } +#endif +} diff --git a/tests/test_type_caster_pyobject_ptr.py b/tests/test_type_caster_pyobject_ptr.py new file mode 100644 index 00000000..1f1ece2b --- /dev/null +++ b/tests/test_type_caster_pyobject_ptr.py @@ -0,0 +1,104 @@ +import pytest + +from pybind11_tests import type_caster_pyobject_ptr as m + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below. +class ValueHolder: + def __init__(self, value): + self.value = value + + +def test_cast_from_pyobject_ptr(): + assert m.cast_from_pyobject_ptr() == 6758 + + +def test_cast_handle_to_pyobject_ptr(): + assert m.cast_handle_to_pyobject_ptr(ValueHolder(24)) == 76 + + +def test_cast_object_to_pyobject_ptr(): + assert m.cast_object_to_pyobject_ptr(ValueHolder(43)) == 257 + + +def test_cast_list_to_pyobject_ptr(): + assert m.cast_list_to_pyobject_ptr([1, 2, 3, 4, 5]) == 395 + + +def test_return_pyobject_ptr(): + assert m.return_pyobject_ptr() == 2314 + + +def test_pass_pyobject_ptr(): + assert m.pass_pyobject_ptr(ValueHolder(82)) == 118 + + +@pytest.mark.parametrize( + "call_callback", + [ + m.call_callback_with_object_return, + m.call_callback_with_pyobject_ptr_return, + ], +) +def test_call_callback_with_object_return(call_callback): + def cb(value): + if value < 0: + raise ValueError("Raised from cb") + return ValueHolder(1000 - value) + + assert call_callback(cb, 287).value == 713 + + with pytest.raises(ValueError, match="^Raised from cb$"): + call_callback(cb, -1) + + +def test_call_callback_with_pyobject_ptr_arg(): + def cb(obj): + return 300 - obj.value + + assert m.call_callback_with_pyobject_ptr_arg(cb, ValueHolder(39)) == 261 + + +@pytest.mark.parametrize("set_error", [True, False]) +def test_cast_to_python_nullptr(set_error): + expected = { + True: r"^Reflective of healthy error handling\.$", + False: ( + r"^Internal error: pybind11::error_already_set called " + r"while Python error indicator not set\.$" + ), + }[set_error] + with pytest.raises(RuntimeError, match=expected): + m.cast_to_pyobject_ptr_nullptr(set_error) + + +def test_cast_to_python_non_nullptr_with_error_set(): + with pytest.raises(SystemError) as excinfo: + m.cast_to_pyobject_ptr_non_nullptr_with_error_set() + assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()" + assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling." + + +def test_pass_list_pyobject_ptr(): + acc = m.pass_list_pyobject_ptr([ValueHolder(842), ValueHolder(452)]) + assert acc == 842452 + + +def test_return_list_pyobject_ptr_take_ownership(): + vec_obj = m.return_list_pyobject_ptr_take_ownership(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + + +def test_return_list_pyobject_ptr_reference(): + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + +def test_type_caster_name_via_incompatible_function_arguments_type_error(): + with pytest.raises(TypeError, match=r"1\. \(arg0: object, arg1: int\) -> None"): + m.pass_pyobject_ptr_and_int(ValueHolder(101), ValueHolder(202)) diff --git a/tests/test_union.py b/tests/test_union.py index 2a2c12fb..e1866e70 100644 --- a/tests/test_union.py +++ b/tests/test_union.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from pybind11_tests import union_ as m diff --git a/tests/test_unnamed_namespace_a.cpp b/tests/test_unnamed_namespace_a.cpp new file mode 100644 index 00000000..2152e64b --- /dev/null +++ b/tests/test_unnamed_namespace_a.cpp @@ -0,0 +1,38 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_a, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_<any_struct>(m, "unnamed_namespace_a_any_struct"); + } else { + m.attr("unnamed_namespace_a_any_struct") = py::none(); + } + m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION; + m.attr("defined_WIN32_or__WIN32") = +#if defined(WIN32) || defined(_WIN32) + true; +#else + false; +#endif + m.attr("defined___clang__") = +#if defined(__clang__) + true; +#else + false; +#endif + m.attr("defined__LIBCPP_VERSION") = +#if defined(_LIBCPP_VERSION) + true; +#else + false; +#endif + m.attr("defined___GLIBCXX__") = +#if defined(__GLIBCXX__) + true; +#else + false; +#endif +} diff --git a/tests/test_unnamed_namespace_a.py b/tests/test_unnamed_namespace_a.py new file mode 100644 index 00000000..9d9856c5 --- /dev/null +++ b/tests/test_unnamed_namespace_a.py @@ -0,0 +1,34 @@ +import pytest + +from pybind11_tests import unnamed_namespace_a as m +from pybind11_tests import unnamed_namespace_b as mb + +XFAIL_CONDITION = ( + "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))" + " or " + "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32" + " and " + "(m.defined___clang__ or m.defined__LIBCPP_VERSION))" +) +XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=False) +@pytest.mark.parametrize( + "any_struct", [m.unnamed_namespace_a_any_struct, mb.unnamed_namespace_b_any_struct] +) +def test_have_class_any_struct(any_struct): + assert any_struct is not None + + +def test_have_at_least_one_class_any_struct(): + assert ( + m.unnamed_namespace_a_any_struct is not None + or mb.unnamed_namespace_b_any_struct is not None + ) + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=True) +def test_have_both_class_any_struct(): + assert m.unnamed_namespace_a_any_struct is not None + assert mb.unnamed_namespace_b_any_struct is not None diff --git a/tests/test_unnamed_namespace_b.cpp b/tests/test_unnamed_namespace_b.cpp new file mode 100644 index 00000000..f97757a7 --- /dev/null +++ b/tests/test_unnamed_namespace_b.cpp @@ -0,0 +1,13 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_b, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_<any_struct>(m, "unnamed_namespace_b_any_struct"); + } else { + m.attr("unnamed_namespace_b_any_struct") = py::none(); + } +} diff --git a/tests/test_unnamed_namespace_b.py b/tests/test_unnamed_namespace_b.py new file mode 100644 index 00000000..4bcaa7a6 --- /dev/null +++ b/tests/test_unnamed_namespace_b.py @@ -0,0 +1,5 @@ +from pybind11_tests import unnamed_namespace_b as m + + +def test_have_attr_any_struct(): + assert hasattr(m, "unnamed_namespace_b_any_struct") diff --git a/tests/test_vector_unique_ptr_member.cpp b/tests/test_vector_unique_ptr_member.cpp new file mode 100644 index 00000000..680cf9a6 --- /dev/null +++ b/tests/test_vector_unique_ptr_member.cpp @@ -0,0 +1,54 @@ +#include "pybind11_tests.h" + +#include <cstddef> +#include <memory> +#include <vector> + +namespace pybind11_tests { +namespace vector_unique_ptr_member { + +struct DataType {}; + +// Reduced from a use case in the wild. +struct VectorOwner { + static std::unique_ptr<VectorOwner> Create(std::size_t num_elems) { + return std::unique_ptr<VectorOwner>( + new VectorOwner(std::vector<std::unique_ptr<DataType>>(num_elems))); + } + + std::size_t data_size() const { return data_.size(); } + +private: + explicit VectorOwner(std::vector<std::unique_ptr<DataType>> data) : data_(std::move(data)) {} + + const std::vector<std::unique_ptr<DataType>> data_; +}; + +} // namespace vector_unique_ptr_member +} // namespace pybind11_tests + +namespace pybind11 { +namespace detail { + +template <> +struct is_copy_constructible<pybind11_tests::vector_unique_ptr_member::VectorOwner> + : std::false_type {}; + +template <> +struct is_move_constructible<pybind11_tests::vector_unique_ptr_member::VectorOwner> + : std::false_type {}; + +} // namespace detail +} // namespace pybind11 + +using namespace pybind11_tests::vector_unique_ptr_member; + +py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } + +TEST_SUBMODULE(vector_unique_ptr_member, m) { + py::class_<VectorOwner>(m, "VectorOwner") + .def_static("Create", &VectorOwner::Create) + .def("data_size", &VectorOwner::data_size); + + m.def("py_cast_VectorOwner_ptr", py_cast_VectorOwner_ptr); +} diff --git a/tests/test_vector_unique_ptr_member.py b/tests/test_vector_unique_ptr_member.py new file mode 100644 index 00000000..2da3d97c --- /dev/null +++ b/tests/test_vector_unique_ptr_member.py @@ -0,0 +1,14 @@ +import pytest + +from pybind11_tests import vector_unique_ptr_member as m + + +@pytest.mark.parametrize("num_elems", range(3)) +def test_create(num_elems): + vo = m.VectorOwner.Create(num_elems) + assert vo.data_size() == num_elems + + +def test_cast(): + vo = m.VectorOwner.Create(0) + assert m.py_cast_VectorOwner_ptr(vo) is vo diff --git a/tests/test_virtual_functions.cpp b/tests/test_virtual_functions.cpp index 685d64a7..93b136ad 100644 --- a/tests/test_virtual_functions.cpp +++ b/tests/test_virtual_functions.cpp @@ -7,22 +7,28 @@ BSD-style license that can be found in the LICENSE file. */ -#include "pybind11_tests.h" -#include "constructor_stats.h" #include <pybind11/functional.h> + +#include "constructor_stats.h" +#include "pybind11_tests.h" + #include <thread> /* This is an example class that we'll want to be able to extend from Python */ -class ExampleVirt { +class ExampleVirt { public: - ExampleVirt(int state) : state(state) { print_created(this, state); } + explicit ExampleVirt(int state) : state(state) { print_created(this, state); } ExampleVirt(const ExampleVirt &e) : state(e.state) { print_copy_created(this); } - ExampleVirt(ExampleVirt &&e) : state(e.state) { print_move_created(this); e.state = 0; } + ExampleVirt(ExampleVirt &&e) noexcept : state(e.state) { + print_move_created(this); + e.state = 0; + } virtual ~ExampleVirt() { print_destroyed(this); } virtual int run(int value) { py::print("Original implementation of " - "ExampleVirt::run(state={}, value={}, str1={}, str2={})"_s.format(state, value, get_string1(), *get_string2())); + "ExampleVirt::run(state={}, value={}, str1={}, str2={})"_s.format( + state, value, get_string1(), *get_string2())); return state + value; } @@ -30,8 +36,8 @@ public: virtual void pure_virtual() = 0; // Returning a reference/pointer to a type converted from python (numbers, strings, etc.) is a - // bit trickier, because the actual int& or std::string& or whatever only exists temporarily, so - // we have to handle it specially in the trampoline class (see below). + // bit trickier, because the actual int& or std::string& or whatever only exists temporarily, + // so we have to handle it specially in the trampoline class (see below). virtual const std::string &get_string1() { return str1; } virtual const std::string *get_string2() { return &str2; } @@ -47,66 +53,63 @@ public: int run(int value) override { /* Generate wrapping code that enables native function overloading */ - PYBIND11_OVERRIDE( - int, /* Return type */ - ExampleVirt, /* Parent class */ - run, /* Name of function */ - value /* Argument(s) */ + PYBIND11_OVERRIDE(int, /* Return type */ + ExampleVirt, /* Parent class */ + run, /* Name of function */ + value /* Argument(s) */ ); } bool run_bool() override { - PYBIND11_OVERRIDE_PURE( - bool, /* Return type */ - ExampleVirt, /* Parent class */ - run_bool, /* Name of function */ - /* This function has no arguments. The trailing comma - in the previous line is needed for some compilers */ + PYBIND11_OVERRIDE_PURE(bool, /* Return type */ + ExampleVirt, /* Parent class */ + run_bool, /* Name of function */ + /* This function has no arguments. The trailing comma + in the previous line is needed for some compilers */ ); } void pure_virtual() override { - PYBIND11_OVERRIDE_PURE( - void, /* Return type */ - ExampleVirt, /* Parent class */ - pure_virtual, /* Name of function */ - /* This function has no arguments. The trailing comma - in the previous line is needed for some compilers */ + PYBIND11_OVERRIDE_PURE(void, /* Return type */ + ExampleVirt, /* Parent class */ + pure_virtual, /* Name of function */ + /* This function has no arguments. The trailing comma + in the previous line is needed for some compilers */ ); } // We can return reference types for compatibility with C++ virtual interfaces that do so, but // note they have some significant limitations (see the documentation). const std::string &get_string1() override { - PYBIND11_OVERRIDE( - const std::string &, /* Return type */ - ExampleVirt, /* Parent class */ - get_string1, /* Name of function */ - /* (no arguments) */ + PYBIND11_OVERRIDE(const std::string &, /* Return type */ + ExampleVirt, /* Parent class */ + get_string1, /* Name of function */ + /* (no arguments) */ ); } const std::string *get_string2() override { - PYBIND11_OVERRIDE( - const std::string *, /* Return type */ - ExampleVirt, /* Parent class */ - get_string2, /* Name of function */ - /* (no arguments) */ + PYBIND11_OVERRIDE(const std::string *, /* Return type */ + ExampleVirt, /* Parent class */ + get_string2, /* Name of function */ + /* (no arguments) */ ); } - }; class NonCopyable { public: - NonCopyable(int a, int b) : value{new int(a*b)} { print_created(this, a, b); } - NonCopyable(NonCopyable &&o) { value = std::move(o.value); print_move_created(this); } + NonCopyable(int a, int b) : value{new int(a * b)} { print_created(this, a, b); } + NonCopyable(NonCopyable &&o) noexcept : value{std::move(o.value)} { print_move_created(this); } NonCopyable(const NonCopyable &) = delete; NonCopyable() = delete; void operator=(const NonCopyable &) = delete; void operator=(NonCopyable &&) = delete; std::string get_value() const { - if (value) return std::to_string(*value); else return "(null)"; + if (value) { + return std::to_string(*value); + } + return "(null)"; } ~NonCopyable() { print_destroyed(this); } @@ -118,11 +121,12 @@ private: // when it is not referenced elsewhere, but copied if it is still referenced. class Movable { public: - Movable(int a, int b) : value{a+b} { print_created(this, a, b); } - Movable(const Movable &m) { value = m.value; print_copy_created(this); } - Movable(Movable &&m) { value = std::move(m.value); print_move_created(this); } + Movable(int a, int b) : value{a + b} { print_created(this, a, b); } + Movable(const Movable &m) : value{m.value} { print_copy_created(this); } + Movable(Movable &&m) noexcept : value{m.value} { print_move_created(this); } std::string get_value() const { return std::to_string(value); } ~Movable() { print_destroyed(this); } + private: int value; }; @@ -131,7 +135,7 @@ class NCVirt { public: virtual ~NCVirt() = default; NCVirt() = default; - NCVirt(const NCVirt&) = delete; + NCVirt(const NCVirt &) = delete; virtual NonCopyable get_noncopyable(int a, int b) { return NonCopyable(a, b); } virtual Movable get_movable(int a, int b) = 0; @@ -150,11 +154,10 @@ class NCVirtTrampoline : public NCVirt { }; struct Base { - /* for some reason MSVC2015 can't compile this if the function is pure virtual */ - virtual std::string dispatch() const { return {}; }; + virtual std::string dispatch() const = 0; virtual ~Base() = default; Base() = default; - Base(const Base&) = delete; + Base(const Base &) = delete; }; struct DispatchIssue : Base { @@ -163,18 +166,38 @@ struct DispatchIssue : Base { } }; +// An abstract adder class that uses visitor pattern to add two data +// objects and send the result to the visitor functor +struct AdderBase { + struct Data {}; + using DataVisitor = std::function<void(const Data &)>; + + virtual void + operator()(const Data &first, const Data &second, const DataVisitor &visitor) const + = 0; + virtual ~AdderBase() = default; + AdderBase() = default; + AdderBase(const AdderBase &) = delete; +}; + +struct Adder : AdderBase { + void + operator()(const Data &first, const Data &second, const DataVisitor &visitor) const override { + PYBIND11_OVERRIDE_PURE_NAME( + void, AdderBase, "__call__", operator(), first, second, visitor); + } +}; + static void test_gil() { { py::gil_scoped_acquire lock; py::print("1st lock acquired"); - } { py::gil_scoped_acquire lock; py::print("2nd lock acquired"); } - } static void test_gil_from_thread() { @@ -184,9 +207,28 @@ static void test_gil_from_thread() { t.join(); } +class test_override_cache_helper { + +public: + virtual int func() { return 0; } + + test_override_cache_helper() = default; + virtual ~test_override_cache_helper() = default; + // Non-copyable + test_override_cache_helper &operator=(test_override_cache_helper const &Right) = delete; + test_override_cache_helper(test_override_cache_helper const &Copy) = delete; +}; + +class test_override_cache_helper_trampoline : public test_override_cache_helper { + int func() override { PYBIND11_OVERRIDE(int, test_override_cache_helper, func); } +}; -// Forward declaration (so that we can put the main tests here; the inherited virtual approaches are -// rather long). +inline int test_override_cache(std::shared_ptr<test_override_cache_helper> const &instance) { + return instance->func(); +} + +// Forward declaration (so that we can put the main tests here; the inherited virtual approaches +// are rather long). void initialize_inherited_virtuals(py::module_ &m); TEST_SUBMODULE(virtual_functions, m) { @@ -198,11 +240,9 @@ TEST_SUBMODULE(virtual_functions, m) { .def("run_bool", &ExampleVirt::run_bool) .def("pure_virtual", &ExampleVirt::pure_virtual); - py::class_<NonCopyable>(m, "NonCopyable") - .def(py::init<int, int>()); + py::class_<NonCopyable>(m, "NonCopyable").def(py::init<int, int>()); - py::class_<Movable>(m, "Movable") - .def(py::init<int, int>()); + py::class_<Movable>(m, "Movable").def(py::init<int, int>()); // test_move_support #if !defined(__INTEL_COMPILER) && !defined(__CUDACC__) && !defined(__PGIC__) @@ -215,7 +255,7 @@ TEST_SUBMODULE(virtual_functions, m) { #endif m.def("runExampleVirt", [](ExampleVirt *ex, int value) { return ex->run(value); }); - m.def("runExampleVirtBool", [](ExampleVirt* ex) { return ex->run_bool(); }); + m.def("runExampleVirtBool", [](ExampleVirt *ex) { return ex->run_bool(); }); m.def("runExampleVirtVirtual", [](ExampleVirt *ex) { ex->pure_virtual(); }); m.def("cstats_debug", &ConstructorStats::get<ExampleVirt>); @@ -226,27 +266,25 @@ TEST_SUBMODULE(virtual_functions, m) { // that were not extended on the Python side struct A { A() = default; - A(const A&) = delete; + A(const A &) = delete; virtual ~A() = default; virtual void f() { py::print("A.f()"); } }; struct PyA : A { PyA() { py::print("PyA.PyA()"); } - PyA(const PyA&) = delete; + PyA(const PyA &) = delete; ~PyA() override { py::print("PyA.~PyA()"); } void f() override { py::print("PyA.f()"); - // This convolution just gives a `void`, but tests that PYBIND11_TYPE() works to protect - // a type containing a , + // This convolution just gives a `void`, but tests that PYBIND11_TYPE() works to + // protect a type containing a , PYBIND11_OVERRIDE(PYBIND11_TYPE(typename std::enable_if<true, void>::type), A, f); } }; - py::class_<A, PyA>(m, "A") - .def(py::init<>()) - .def("f", &A::f); + py::class_<A, PyA>(m, "A").def(py::init<>()).def("f", &A::f); m.def("call_f", [](A *a) { a->f(); }); @@ -254,14 +292,14 @@ TEST_SUBMODULE(virtual_functions, m) { // ... unless we explicitly request it, as in this example: struct A2 { A2() = default; - A2(const A2&) = delete; + A2(const A2 &) = delete; virtual ~A2() = default; virtual void f() { py::print("A2.f()"); } }; struct PyA2 : A2 { PyA2() { py::print("PyA2.PyA2()"); } - PyA2(const PyA2&) = delete; + PyA2(const PyA2 &) = delete; ~PyA2() override { py::print("PyA2.~PyA2()"); } void f() override { py::print("PyA2.f()"); @@ -282,18 +320,46 @@ TEST_SUBMODULE(virtual_functions, m) { .def(py::init<>()) .def("dispatch", &Base::dispatch); - m.def("dispatch_issue_go", [](const Base * b) { return b->dispatch(); }); + m.def("dispatch_issue_go", [](const Base *b) { return b->dispatch(); }); + + // test_recursive_dispatch_issue + // #3357: Recursive dispatch fails to find python function override + pybind11::class_<AdderBase, Adder>(m, "Adder") + .def(pybind11::init<>()) + .def("__call__", &AdderBase::operator()); + + pybind11::class_<AdderBase::Data>(m, "Data").def(pybind11::init<>()); + + m.def("add2", + [](const AdderBase::Data &first, + const AdderBase::Data &second, + const AdderBase &adder, + const AdderBase::DataVisitor &visitor) { adder(first, second, visitor); }); + + m.def("add3", + [](const AdderBase::Data &first, + const AdderBase::Data &second, + const AdderBase::Data &third, + const AdderBase &adder, + const AdderBase::DataVisitor &visitor) { + adder(first, second, [&](const AdderBase::Data &first_plus_second) { + // NOLINTNEXTLINE(readability-suspicious-call-argument) + adder(first_plus_second, third, visitor); + }); + }); // test_override_ref // #392/397: overriding reference-returning functions class OverrideTest { public: - struct A { std::string value = "hi"; }; + struct A { + std::string value = "hi"; + }; std::string v; A a; explicit OverrideTest(const std::string &v) : v{v} {} OverrideTest() = default; - OverrideTest(const OverrideTest&) = delete; + OverrideTest(const OverrideTest &) = delete; virtual std::string str_value() { return v; } virtual std::string &str_ref() { return v; } virtual A A_value() { return a; } @@ -304,14 +370,22 @@ TEST_SUBMODULE(virtual_functions, m) { class PyOverrideTest : public OverrideTest { public: using OverrideTest::OverrideTest; - std::string str_value() override { PYBIND11_OVERRIDE(std::string, OverrideTest, str_value); } - // Not allowed (uncommenting should hit a static_assert failure): we can't get a reference - // to a python numeric value, since we only copy values in the numeric type caster: -// std::string &str_ref() override { PYBIND11_OVERRIDE(std::string &, OverrideTest, str_ref); } + std::string str_value() override { + PYBIND11_OVERRIDE(std::string, OverrideTest, str_value); + } + // Not allowed (enabling the below should hit a static_assert failure): we can't get a + // reference to a python numeric value, since we only copy values in the numeric type + // caster: +#ifdef PYBIND11_NEVER_DEFINED_EVER + std::string &str_ref() override { + PYBIND11_OVERRIDE(std::string &, OverrideTest, str_ref); + } +#endif // But we can work around it like this: private: std::string _tmp; std::string str_ref_helper() { PYBIND11_OVERRIDE(std::string, OverrideTest, str_ref); } + public: std::string &str_ref() override { return _tmp = str_ref_helper(); } @@ -324,11 +398,20 @@ TEST_SUBMODULE(virtual_functions, m) { py::class_<OverrideTest, PyOverrideTest>(m, "OverrideTest") .def(py::init<const std::string &>()) .def("str_value", &OverrideTest::str_value) -// .def("str_ref", &OverrideTest::str_ref) +#ifdef PYBIND11_NEVER_DEFINED_EVER + .def("str_ref", &OverrideTest::str_ref) +#endif .def("A_value", &OverrideTest::A_value) .def("A_ref", &OverrideTest::A_ref); -} + py::class_<test_override_cache_helper, + test_override_cache_helper_trampoline, + std::shared_ptr<test_override_cache_helper>>(m, "test_override_cache_helper") + .def(py::init_alias<>()) + .def("func", &test_override_cache_helper::func); + + m.def("test_override_cache", test_override_cache); +} // Inheriting virtual methods. We do two versions here: the repeat-everything version and the // templated trampoline versions mentioned in docs/advanced.rst. @@ -338,94 +421,107 @@ TEST_SUBMODULE(virtual_functions, m) { // properly (pybind11, sensibly, doesn't allow us to bind the same C++ class to // multiple python classes). class A_Repeat { -#define A_METHODS \ -public: \ - virtual int unlucky_number() = 0; \ - virtual std::string say_something(unsigned times) { \ - std::string s = ""; \ - for (unsigned i = 0; i < times; ++i) \ - s += "hi"; \ - return s; \ - } \ - std::string say_everything() { \ - return say_something(1) + " " + std::to_string(unlucky_number()); \ +#define A_METHODS \ +public: \ + virtual int unlucky_number() = 0; \ + virtual std::string say_something(unsigned times) { \ + std::string s = ""; \ + for (unsigned i = 0; i < times; ++i) \ + s += "hi"; \ + return s; \ + } \ + std::string say_everything() { \ + return say_something(1) + " " + std::to_string(unlucky_number()); \ } -A_METHODS + A_METHODS A_Repeat() = default; - A_Repeat(const A_Repeat&) = delete; + A_Repeat(const A_Repeat &) = delete; virtual ~A_Repeat() = default; }; class B_Repeat : public A_Repeat { -#define B_METHODS \ -public: \ - int unlucky_number() override { return 13; } \ - std::string say_something(unsigned times) override { \ - return "B says hi " + std::to_string(times) + " times"; \ - } \ +#define B_METHODS \ +public: \ + int unlucky_number() override { return 13; } \ + std::string say_something(unsigned times) override { \ + return "B says hi " + std::to_string(times) + " times"; \ + } \ virtual double lucky_number() { return 7.0; } -B_METHODS + B_METHODS }; class C_Repeat : public B_Repeat { -#define C_METHODS \ -public: \ - int unlucky_number() override { return 4444; } \ +#define C_METHODS \ +public: \ + int unlucky_number() override { return 4444; } \ double lucky_number() override { return 888; } -C_METHODS + C_METHODS }; class D_Repeat : public C_Repeat { #define D_METHODS // Nothing overridden. -D_METHODS + D_METHODS }; // Base classes for templated inheritance trampolines. Identical to the repeat-everything version: class A_Tpl { A_METHODS; A_Tpl() = default; - A_Tpl(const A_Tpl&) = delete; + A_Tpl(const A_Tpl &) = delete; virtual ~A_Tpl() = default; }; -class B_Tpl : public A_Tpl { B_METHODS }; -class C_Tpl : public B_Tpl { C_METHODS }; -class D_Tpl : public C_Tpl { D_METHODS }; - +class B_Tpl : public A_Tpl { + B_METHODS +}; +class C_Tpl : public B_Tpl { + C_METHODS +}; +class D_Tpl : public C_Tpl { + D_METHODS +}; // Inheritance approach 1: each trampoline gets every virtual method (11 in total) class PyA_Repeat : public A_Repeat { public: using A_Repeat::A_Repeat; int unlucky_number() override { PYBIND11_OVERRIDE_PURE(int, A_Repeat, unlucky_number, ); } - std::string say_something(unsigned times) override { PYBIND11_OVERRIDE(std::string, A_Repeat, say_something, times); } + std::string say_something(unsigned times) override { + PYBIND11_OVERRIDE(std::string, A_Repeat, say_something, times); + } }; class PyB_Repeat : public B_Repeat { public: using B_Repeat::B_Repeat; int unlucky_number() override { PYBIND11_OVERRIDE(int, B_Repeat, unlucky_number, ); } - std::string say_something(unsigned times) override { PYBIND11_OVERRIDE(std::string, B_Repeat, say_something, times); } + std::string say_something(unsigned times) override { + PYBIND11_OVERRIDE(std::string, B_Repeat, say_something, times); + } double lucky_number() override { PYBIND11_OVERRIDE(double, B_Repeat, lucky_number, ); } }; class PyC_Repeat : public C_Repeat { public: using C_Repeat::C_Repeat; int unlucky_number() override { PYBIND11_OVERRIDE(int, C_Repeat, unlucky_number, ); } - std::string say_something(unsigned times) override { PYBIND11_OVERRIDE(std::string, C_Repeat, say_something, times); } + std::string say_something(unsigned times) override { + PYBIND11_OVERRIDE(std::string, C_Repeat, say_something, times); + } double lucky_number() override { PYBIND11_OVERRIDE(double, C_Repeat, lucky_number, ); } }; class PyD_Repeat : public D_Repeat { public: using D_Repeat::D_Repeat; int unlucky_number() override { PYBIND11_OVERRIDE(int, D_Repeat, unlucky_number, ); } - std::string say_something(unsigned times) override { PYBIND11_OVERRIDE(std::string, D_Repeat, say_something, times); } + std::string say_something(unsigned times) override { + PYBIND11_OVERRIDE(std::string, D_Repeat, say_something, times); + } double lucky_number() override { PYBIND11_OVERRIDE(double, D_Repeat, lucky_number, ); } }; // Inheritance approach 2: templated trampoline classes. // // Advantages: -// - we have only 2 (template) class and 4 method declarations (one per virtual method, plus one for -// any override of a pure virtual method), versus 4 classes and 6 methods (MI) or 4 classes and 11 -// methods (repeat). -// - Compared to MI, we also don't have to change the non-trampoline inheritance to virtual, and can -// properly inherit constructors. +// - we have only 2 (template) class and 4 method declarations (one per virtual method, plus one +// for any override of a pure virtual method), versus 4 classes and 6 methods (MI) or 4 classes +// and 11 methods (repeat). +// - Compared to MI, we also don't have to change the non-trampoline inheritance to virtual, and +// can properly inherit constructors. // // Disadvantage: // - the compiler must still generate and compile 14 different methods (more, even, than the 11 @@ -437,17 +533,20 @@ class PyA_Tpl : public Base { public: using Base::Base; // Inherit constructors int unlucky_number() override { PYBIND11_OVERRIDE_PURE(int, Base, unlucky_number, ); } - std::string say_something(unsigned times) override { PYBIND11_OVERRIDE(std::string, Base, say_something, times); } + std::string say_something(unsigned times) override { + PYBIND11_OVERRIDE(std::string, Base, say_something, times); + } }; template <class Base = B_Tpl> class PyB_Tpl : public PyA_Tpl<Base> { public: using PyA_Tpl<Base>::PyA_Tpl; // Inherit constructors (via PyA_Tpl's inherited constructors) + // NOLINTNEXTLINE(bugprone-parent-virtual-call) int unlucky_number() override { PYBIND11_OVERRIDE(int, Base, unlucky_number, ); } double lucky_number() override { PYBIND11_OVERRIDE(double, Base, lucky_number, ); } }; -// Since C_Tpl and D_Tpl don't declare any new virtual methods, we don't actually need these (we can -// use PyB_Tpl<C_Tpl> and PyB_Tpl<D_Tpl> for the trampoline classes instead): +// Since C_Tpl and D_Tpl don't declare any new virtual methods, we don't actually need these +// (we can use PyB_Tpl<C_Tpl> and PyB_Tpl<D_Tpl> for the trampoline classes instead): /* template <class Base = C_Tpl> class PyC_Tpl : public PyB_Tpl<Base> { public: @@ -471,10 +570,8 @@ void initialize_inherited_virtuals(py::module_ &m) { py::class_<B_Repeat, A_Repeat, PyB_Repeat>(m, "B_Repeat") .def(py::init<>()) .def("lucky_number", &B_Repeat::lucky_number); - py::class_<C_Repeat, B_Repeat, PyC_Repeat>(m, "C_Repeat") - .def(py::init<>()); - py::class_<D_Repeat, C_Repeat, PyD_Repeat>(m, "D_Repeat") - .def(py::init<>()); + py::class_<C_Repeat, B_Repeat, PyC_Repeat>(m, "C_Repeat").def(py::init<>()); + py::class_<D_Repeat, C_Repeat, PyD_Repeat>(m, "D_Repeat").def(py::init<>()); // test_ // Method 2: Templated trampolines @@ -486,11 +583,8 @@ void initialize_inherited_virtuals(py::module_ &m) { py::class_<B_Tpl, A_Tpl, PyB_Tpl<>>(m, "B_Tpl") .def(py::init<>()) .def("lucky_number", &B_Tpl::lucky_number); - py::class_<C_Tpl, B_Tpl, PyB_Tpl<C_Tpl>>(m, "C_Tpl") - .def(py::init<>()); - py::class_<D_Tpl, C_Tpl, PyB_Tpl<D_Tpl>>(m, "D_Tpl") - .def(py::init<>()); - + py::class_<C_Tpl, B_Tpl, PyB_Tpl<C_Tpl>>(m, "C_Tpl").def(py::init<>()); + py::class_<D_Tpl, C_Tpl, PyB_Tpl<D_Tpl>>(m, "D_Tpl").def(py::init<>()); // Fix issue #1454 (crash when acquiring/releasing GIL on another thread in Python 2.7) m.def("test_gil", &test_gil); diff --git a/tests/test_virtual_functions.py b/tests/test_virtual_functions.py index f7d3bd1e..c17af7df 100644 --- a/tests/test_virtual_functions.py +++ b/tests/test_virtual_functions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import pytest import env # noqa: F401 @@ -10,12 +9,12 @@ from pybind11_tests import ConstructorStats # noqa: E402 def test_override(capture, msg): class ExtendedExampleVirt(m.ExampleVirt): def __init__(self, state): - super(ExtendedExampleVirt, self).__init__(state + 1) + super().__init__(state + 1) self.data = "Hello world" def run(self, value): - print("ExtendedExampleVirt::run(%i), calling parent.." % value) - return super(ExtendedExampleVirt, self).run(value + 1) + print(f"ExtendedExampleVirt::run({value}), calling parent..") + return super().run(value + 1) def run_bool(self): print("ExtendedExampleVirt::run_bool()") @@ -25,11 +24,11 @@ def test_override(capture, msg): return "override1" def pure_virtual(self): - print("ExtendedExampleVirt::pure_virtual(): %s" % self.data) + print(f"ExtendedExampleVirt::pure_virtual(): {self.data}") class ExtendedExampleVirt2(ExtendedExampleVirt): def __init__(self, state): - super(ExtendedExampleVirt2, self).__init__(state + 1) + super().__init__(state + 1) def get_string2(self): return "override2" @@ -41,7 +40,7 @@ def test_override(capture, msg): capture == """ Original implementation of ExampleVirt::run(state=10, value=20, str1=default1, str2=default2) - """ # noqa: E501 line too long + """ ) with pytest.raises(RuntimeError) as excinfo: @@ -59,7 +58,7 @@ def test_override(capture, msg): == """ ExtendedExampleVirt::run(20), calling parent.. Original implementation of ExampleVirt::run(state=11, value=21, str1=override1, str2=default2) - """ # noqa: E501 line too long + """ ) with capture: assert m.runExampleVirtBool(ex12p) is False @@ -76,7 +75,7 @@ def test_override(capture, msg): == """ ExtendedExampleVirt::run(50), calling parent.. Original implementation of ExampleVirt::run(state=17, value=51, str1=override1, str2=override2) - """ # noqa: E501 line too long + """ ) cstats = ConstructorStats.get(m.ExampleVirt) @@ -97,7 +96,7 @@ def test_alias_delay_initialization1(capture): class B(m.A): def __init__(self): - super(B, self).__init__() + super().__init__() def f(self): print("In python f()") @@ -137,7 +136,7 @@ def test_alias_delay_initialization2(capture): class B2(m.A2): def __init__(self): - super(B2, self).__init__() + super().__init__() def f(self): print("In python B2.f()") @@ -193,8 +192,7 @@ def test_move_support(): class NCVirtExt(m.NCVirt): def get_noncopyable(self, a, b): # Constructs and returns a new instance: - nc = m.NonCopyable(a * a, b * b) - return nc + return m.NonCopyable(a * a, b * b) def get_movable(self, a, b): # Return a referenced copy @@ -245,7 +243,7 @@ def test_dispatch_issue(msg): class PyClass2(m.DispatchIssue): def dispatch(self): with pytest.raises(RuntimeError) as excinfo: - super(PyClass2, self).dispatch() + super().dispatch() assert ( msg(excinfo.value) == 'Tried to call pure virtual function "Base::dispatch"' @@ -257,6 +255,39 @@ def test_dispatch_issue(msg): assert m.dispatch_issue_go(b) == "Yay.." +def test_recursive_dispatch_issue(): + """#3357: Recursive dispatch fails to find python function override""" + + class Data(m.Data): + def __init__(self, value): + super().__init__() + self.value = value + + class Adder(m.Adder): + def __call__(self, first, second, visitor): + # lambda is a workaround, which adds extra frame to the + # current CPython thread. Removing lambda reveals the bug + # [https://github.com/pybind/pybind11/issues/3357] + (lambda: visitor(Data(first.value + second.value)))() # noqa: PLC3002 + + class StoreResultVisitor: + def __init__(self): + self.result = None + + def __call__(self, data): + self.result = data.value + + store = StoreResultVisitor() + + m.add2(Data(1), Data(2), Adder(), store) + assert store.result == 3 + + # without lambda in Adder class, this function fails with + # RuntimeError: Tried to call pure virtual function "AdderBase::__call__" + m.add3(Data(1), Data(2), Data(3), Adder(), store) + assert store.result == 6 + + def test_override_ref(): """#392/397: overriding reference-returning functions""" o = m.OverrideTest("asdf") @@ -406,3 +437,22 @@ def test_issue_1454(): # Fix issue #1454 (crash when acquiring/releasing GIL on another thread in Python 2.7) m.test_gil() m.test_gil_from_thread() + + +def test_python_override(): + def func(): + class Test(m.test_override_cache_helper): + def func(self): + return 42 + + return Test() + + def func2(): + class Test(m.test_override_cache_helper): + pass + + return Test() + + for _ in range(1500): + assert m.test_override_cache(func()) == 42 + assert m.test_override_cache(func2()) == 0 diff --git a/tests/valgrind-numpy-scipy.supp b/tests/valgrind-numpy-scipy.supp index 60e9f479..16db302c 100644 --- a/tests/valgrind-numpy-scipy.supp +++ b/tests/valgrind-numpy-scipy.supp @@ -1,4 +1,8 @@ # Valgrind suppression file for NumPy & SciPy errors and leaks in pybind11 tests +# +# On updating a dependency, to get a list of "default" leaks in e.g. NumPy, run +# `PYTHONMALLOC=malloc valgrind --leak-check=full --show-leak-kinds=definite,indirect python3.9-dbg -c "import numpy"` +# To use these suppression files, add e.g. `--suppressions=valgrind-numpy-scipy.supp` { Leaks when importing NumPy @@ -29,7 +33,7 @@ } { - Leaks when importing NumPy (tris) + Leaks when importing NumPy (ter) Memcheck:Leak fun:malloc fun:_PyMem_RawMalloc @@ -59,6 +63,24 @@ fun:_PyObject_GC_Malloc fun:_PyObject_GC_NewVar fun:tuple_alloc + fun:_PyTuple_FromArray + fun:_PyObject_MakeTpCall + fun:_PyObject_VectorcallTstate + fun:_PyObject_CallFunctionVa + fun:PyObject_CallFunction + fun:PyImport_Import +} + +{ + Leaks when importing NumPy (quinquies) + Memcheck:Leak + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + fun:_PyObject_GC_Alloc + fun:_PyObject_GC_Malloc + fun:_PyObject_GC_NewVar + fun:tuple_alloc fun:PyTuple_New fun:r_object fun:r_object @@ -67,7 +89,7 @@ } { - Leaks when importing NumPy (quinquies) + Leaks when importing NumPy (sexies) Memcheck:Leak fun:malloc fun:_PyMem_RawMalloc @@ -89,7 +111,7 @@ fun:_Znwm fun:PyInit_pypocketfft fun:_PyImport_LoadDynamicModuleWithSpec - fun:_imp_create_dynamic_impl.constprop.3 + fun:_imp_create_dynamic_impl* fun:_imp_create_dynamic fun:cfunction_vectorcall_FASTCALL fun:PyVectorcall_Call diff --git a/tests/valgrind-python.supp b/tests/valgrind-python.supp index 1dd04fa7..d77d5e5c 100644 --- a/tests/valgrind-python.supp +++ b/tests/valgrind-python.supp @@ -115,21 +115,3 @@ fun:_PyImport_FindSharedFuncptr fun:_PyImport_LoadDynamicModuleWithSpec } - -# Not really CPython-specific, see link -{ - dlopen leak (https://stackoverflow.com/questions/1542457/memory-leak-reported-by-valgrind-in-dlopen) - Memcheck:Leak - fun:malloc - ... - fun:dl_open_worker - fun:_dl_catch_exception - fun:_dl_open - fun:dlopen_doit - fun:_dl_catch_exception - fun:_dl_catch_error - fun:_dlerror_run - fun:dlopen@@GLIBC_2.2.5 - fun:_PyImport_FindSharedFuncptr - fun:_PyImport_LoadDynamicModuleWithSpec -} diff --git a/tools/FindCatch.cmake b/tools/FindCatch.cmake index 4d6bffcf..5d3fcbfb 100644 --- a/tools/FindCatch.cmake +++ b/tools/FindCatch.cmake @@ -9,6 +9,8 @@ # CATCH_INCLUDE_DIR - path to catch.hpp # CATCH_VERSION - version number +option(DOWNLOAD_CATCH "Download catch2 if not found") + if(NOT Catch_FIND_VERSION) message(FATAL_ERROR "A version number must be specified.") elseif(Catch_FIND_REQUIRED) @@ -34,10 +36,14 @@ endfunction() function(_download_catch version destination_dir) message(STATUS "Downloading catch v${version}...") set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp) - file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status) + file( + DOWNLOAD ${url} "${destination_dir}/catch.hpp" + STATUS status + LOG log) list(GET status 0 error) if(error) - message(FATAL_ERROR "Could not download ${url}") + string(REPLACE "\n" "\n " log " ${log}") + message(FATAL_ERROR "Could not download URL:\n" " ${url}\n" "Log:\n" "${log}") endif() set(CATCH_INCLUDE_DIR "${destination_dir}" diff --git a/tools/FindPythonLibsNew.cmake b/tools/FindPythonLibsNew.cmake index 3605aebc..ce558d4e 100644 --- a/tools/FindPythonLibsNew.cmake +++ b/tools/FindPythonLibsNew.cmake @@ -92,7 +92,7 @@ endif() # Use the Python interpreter to find the libs. if(NOT PythonLibsNew_FIND_VERSION) - set(PythonLibsNew_FIND_VERSION "") + set(PythonLibsNew_FIND_VERSION "3.6") endif() find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} ${_pythonlibs_required} @@ -112,12 +112,26 @@ endif() # VERSION. VERSION will typically be like "2.7" on unix, and "27" on windows. execute_process( COMMAND - "${PYTHON_EXECUTABLE}" "-c" "from distutils import sysconfig as s;import sys;import struct; + "${PYTHON_EXECUTABLE}" "-c" " +import sys;import struct; +import sysconfig as s +USE_SYSCONFIG = sys.version_info >= (3, 10) +if not USE_SYSCONFIG: + from distutils import sysconfig as ds print('.'.join(str(v) for v in sys.version_info)); print(sys.prefix); -print(s.get_python_inc(plat_specific=True)); -print(s.get_python_lib(plat_specific=True)); -print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO')); +if USE_SYSCONFIG: + scheme = s.get_default_scheme() + if scheme == 'posix_local': + # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/ + scheme = 'posix_prefix' + print(s.get_path('platinclude', scheme)) + print(s.get_path('platlib')) + print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO')) +else: + print(ds.get_python_inc(plat_specific=True)); + print(ds.get_python_lib(plat_specific=True)); + print(ds.get_config_var('EXT_SUFFIX') or ds.get_config_var('SO')); print(hasattr(sys, 'gettotalrefcount')+0); print(struct.calcsize('@P')); print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION')); @@ -137,26 +151,40 @@ if(NOT _PYTHON_SUCCESS MATCHES 0) return() endif() +option( + PYBIND11_PYTHONLIBS_OVERWRITE + "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values." + ON) +# Can manually set values when cross-compiling +macro(_PYBIND11_GET_IF_UNDEF lst index name) + if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}") + list(GET "${lst}" "${index}" "${name}") + endif() +endmacro() + # Convert the process output into a list if(WIN32) string(REGEX REPLACE "\\\\" "/" _PYTHON_VALUES ${_PYTHON_VALUES}) endif() string(REGEX REPLACE ";" "\\\\;" _PYTHON_VALUES ${_PYTHON_VALUES}) string(REGEX REPLACE "\n" ";" _PYTHON_VALUES ${_PYTHON_VALUES}) -list(GET _PYTHON_VALUES 0 _PYTHON_VERSION_LIST) -list(GET _PYTHON_VALUES 1 PYTHON_PREFIX) -list(GET _PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) -list(GET _PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) -list(GET _PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) -list(GET _PYTHON_VALUES 5 PYTHON_IS_DEBUG) -list(GET _PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) -list(GET _PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) -list(GET _PYTHON_VALUES 8 PYTHON_LIBDIR) -list(GET _PYTHON_VALUES 9 PYTHON_MULTIARCH) +_pybind11_get_if_undef(_PYTHON_VALUES 0 _PYTHON_VERSION_LIST) +_pybind11_get_if_undef(_PYTHON_VALUES 1 PYTHON_PREFIX) +_pybind11_get_if_undef(_PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) +_pybind11_get_if_undef(_PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) +_pybind11_get_if_undef(_PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) +_pybind11_get_if_undef(_PYTHON_VALUES 5 PYTHON_IS_DEBUG) +_pybind11_get_if_undef(_PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) +_pybind11_get_if_undef(_PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) +_pybind11_get_if_undef(_PYTHON_VALUES 8 PYTHON_LIBDIR) +_pybind11_get_if_undef(_PYTHON_VALUES 9 PYTHON_MULTIARCH) # Make sure the Python has the same pointer-size as the chosen compiler # Skip if CMAKE_SIZEOF_VOID_P is not defined -if(CMAKE_SIZEOF_VOID_P AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}")) +# This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN) +if(NOT CMAKE_CROSSCOMPILING + AND CMAKE_SIZEOF_VOID_P + AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}")) if(PythonLibsNew_FIND_REQUIRED) math(EXPR _PYTHON_BITS "${PYTHON_SIZEOF_VOID_P} * 8") math(EXPR _CMAKE_BITS "${CMAKE_SIZEOF_VOID_P} * 8") @@ -180,7 +208,9 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX "${PYTHON_PREFIX}") string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES "${PYTHON_SITE_PACKAGES}") -if(CMAKE_HOST_WIN32) +if(DEFINED PYTHON_LIBRARY) + # Don't write to PYTHON_LIBRARY if it's already set +elseif(CMAKE_HOST_WIN32) set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/python${PYTHON_LIBRARY_SUFFIX}.lib") # when run in a venv, PYTHON_PREFIX points to it. But the libraries remain in the @@ -246,7 +276,7 @@ if(NOT PYTHON_DEBUG_LIBRARY) endif() set(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}") -find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARY}" +find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARIES}" "${PYTHON_EXECUTABLE}${PYTHON_VERSION_STRING}") set(PYTHONLIBS_FOUND TRUE) diff --git a/tools/JoinPaths.cmake b/tools/JoinPaths.cmake new file mode 100644 index 00000000..c68d91b8 --- /dev/null +++ b/tools/JoinPaths.cmake @@ -0,0 +1,23 @@ +# This module provides function for joining paths +# known from most languages +# +# SPDX-License-Identifier: (MIT OR CC0-1.0) +# Copyright 2020 Jan Tojnar +# https://github.com/jtojnar/cmake-snips +# +# Modelled after Python’s os.path.join +# https://docs.python.org/3.7/library/os.path.html#os.path.join +# Windows not supported +function(join_paths joined_path first_path_segment) + set(temp_path "${first_path_segment}") + foreach(current_segment IN LISTS ARGN) + if(NOT ("${current_segment}" STREQUAL "")) + if(IS_ABSOLUTE "${current_segment}") + set(temp_path "${current_segment}") + else() + set(temp_path "${temp_path}/${current_segment}") + endif() + endif() + endforeach() + set(${joined_path} "${temp_path}" PARENT_SCOPE) +endfunction() diff --git a/tools/check-style.sh b/tools/check-style.sh index f7af2a41..6d832523 100755 --- a/tools/check-style.sh +++ b/tools/check-style.sh @@ -16,11 +16,11 @@ check_style_errors=0 IFS=$'\n' -found="$(grep '\<\(if\|for\|while\|catch\)(\|){' $@ -rn --color=always)" +found="$(grep '\<\(if\|for\|while\|catch\)(\|){' "$@" -rn --color=always)" if [ -n "$found" ]; then echo -e '\033[31;01mError: found the following coding style problems:\033[0m' check_style_errors=1 - echo "$found" | sed -e 's/^/ /' + echo "${found//^/ /}" fi found="$(awk ' @@ -34,7 +34,7 @@ last && /^\s*{/ { last="" } { last = /(if|for|while|catch|switch)\s*\(.*\)\s*$/ ? $0 : "" } -' $(find include -type f) $@)" +' "$(find include -type f)" "$@")" if [ -n "$found" ]; then check_style_errors=1 echo -e '\033[31;01mError: braces should occur on the same line as the if/while/.. statement. Found issues in the following files:\033[0m' diff --git a/tools/codespell_ignore_lines_from_errors.py b/tools/codespell_ignore_lines_from_errors.py new file mode 100644 index 00000000..4ec9add1 --- /dev/null +++ b/tools/codespell_ignore_lines_from_errors.py @@ -0,0 +1,39 @@ +"""Simple script for rebuilding .codespell-ignore-lines + +Usage: + +cat < /dev/null > .codespell-ignore-lines +pre-commit run --all-files codespell >& /tmp/codespell_errors.txt +python3 tools/codespell_ignore_lines_from_errors.py /tmp/codespell_errors.txt > .codespell-ignore-lines + +git diff to review changes, then commit, push. +""" + +import sys +from typing import List + + +def run(args: List[str]) -> None: + assert len(args) == 1, "codespell_errors.txt" + cache = {} + done = set() + with open(args[0]) as f: + lines = f.read().splitlines() + + for line in sorted(lines): + i = line.find(" ==> ") + if i > 0: + flds = line[:i].split(":") + if len(flds) >= 2: + filename, line_num = flds[:2] + if filename not in cache: + with open(filename) as f: + cache[filename] = f.read().splitlines() + supp = cache[filename][int(line_num) - 1] + if supp not in done: + print(supp) + done.add(supp) + + +if __name__ == "__main__": + run(args=sys.argv[1:]) diff --git a/tools/libsize.py b/tools/libsize.py index 589c317f..1ac9afbe 100644 --- a/tools/libsize.py +++ b/tools/libsize.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import print_function, division import os import sys @@ -15,7 +13,7 @@ lib = sys.argv[1] save = sys.argv[2] if not os.path.exists(lib): - sys.exit("Error: requested file ({}) does not exist".format(lib)) + sys.exit(f"Error: requested file ({lib}) does not exist") libsize = os.path.getsize(lib) @@ -30,7 +28,7 @@ if os.path.exists(save): if change == 0: print(" (no change)") else: - print(" (change of {:+} bytes = {:+.2%})".format(change, change / oldsize)) + print(f" (change of {change:+} bytes = {change / oldsize:+.2%})") else: print() diff --git a/tools/make_changelog.py b/tools/make_changelog.py index 609ce2f1..b5bd8329 100755 --- a/tools/make_changelog.py +++ b/tools/make_changelog.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- import re import ghapi.all - from rich import print from rich.syntax import Syntax - ENTRY = re.compile( r""" Suggested \s changelog \s entry: @@ -27,12 +24,17 @@ print() api = ghapi.all.GhApi(owner="pybind", repo="pybind11") -issues = api.issues.list_for_repo(labels="needs changelog", state="closed") +issues_pages = ghapi.page.paged( + api.issues.list_for_repo, labels="needs changelog", state="closed" +) +issues = (issue for page in issues_pages for issue in page) missing = [] for issue in issues: - changelog = ENTRY.findall(issue.body) - if changelog: + changelog = ENTRY.findall(issue.body or "") + if not changelog or not changelog[0]: + missing.append(issue) + else: (msg,) = changelog if not msg.startswith("* "): msg = "* " + msg @@ -41,12 +43,9 @@ for issue in issues: msg += f"\n `#{issue.number} <{issue.html_url}>`_" - print(Syntax(msg, "rst", theme="ansi_light")) + print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) print() - else: - missing.append(issue) - if missing: print() print("[blue]" + "-" * 30) diff --git a/tools/pybind11.pc.in b/tools/pybind11.pc.in new file mode 100644 index 00000000..402f0b35 --- /dev/null +++ b/tools/pybind11.pc.in @@ -0,0 +1,7 @@ +prefix=@prefix_for_pc_file@ +includedir=@includedir_for_pc_file@ + +Name: @PROJECT_NAME@ +Description: Seamless operability between C++11 and Python +Version: @PROJECT_VERSION@ +Cflags: -I${includedir} diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 3c05c682..308d1b70 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -5,10 +5,9 @@ Adds the following targets:: pybind11::pybind11 - link to headers and pybind11 pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries - pybind11::python2_no_register - Avoid warning/error with Python 2 + C++14/7 pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -20,7 +19,8 @@ Adds the following functions:: #]======================================================] # CMake 3.10 has an include_guard command, but we can't use that yet -if(TARGET pybind11::lto) +# include_guard(global) (pre-CMake 3.10) +if(TARGET pybind11::pybind11) return() endif() @@ -65,31 +65,6 @@ set_property( APPEND PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11) -# ----------------------- no register ---------------------- - -# Workaround for Python 2.7 and C++17 (C++14 as a warning) incompatibility -# This adds the flags -Wno-register and -Wno-deprecated-register if the compiler -# is Clang 3.9+ or AppleClang and the compile language is CXX, or /wd5033 for MSVC (all languages, -# since MSVC didn't recognize COMPILE_LANGUAGE until CMake 3.11+). - -add_library(pybind11::python2_no_register INTERFACE IMPORTED ${optional_global}) -set(clang_4plus - "$<AND:$<CXX_COMPILER_ID:Clang>,$<NOT:$<VERSION_LESS:$<CXX_COMPILER_VERSION>,3.9>>>") -set(no_register "$<OR:${clang_4plus},$<CXX_COMPILER_ID:AppleClang>>") - -if(MSVC AND CMAKE_VERSION VERSION_LESS 3.11) - set(cxx_no_register "${no_register}") -else() - set(cxx_no_register "$<AND:$<COMPILE_LANGUAGE:CXX>,${no_register}>") -endif() - -set(msvc "$<CXX_COMPILER_ID:MSVC>") - -set_property( - TARGET pybind11::python2_no_register - PROPERTY INTERFACE_COMPILE_OPTIONS - "$<${cxx_no_register}:-Wno-register;-Wno-deprecated-register>" "$<${msvc}:/wd5033>") - # --------------------------- link helper --------------------------- add_library(pybind11::python_link_helper IMPORTED INTERFACE ${optional_global}) @@ -115,28 +90,32 @@ endif() add_library(pybind11::windows_extras IMPORTED INTERFACE ${optional_global}) -if(MSVC) - # /MP enables multithreaded builds (relevant when there are many files), /bigobj is - # needed for bigger binding projects due to the limit to 64k addressable sections +if(MSVC) # That's also clang-cl + # /bigobj is needed for bigger binding projects due to the limit to 64k + # addressable sections set_property( TARGET pybind11::windows_extras APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS /bigobj) + PROPERTY INTERFACE_COMPILE_OPTIONS $<$<COMPILE_LANGUAGE:CXX>:/bigobj>) - if(CMAKE_VERSION VERSION_LESS 3.11) - set_property( - TARGET pybind11::windows_extras - APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS $<$<NOT:$<CONFIG:Debug>>:/MP>) - else() - # Only set these options for C++ files. This is important so that, for - # instance, projects that include other types of source files like CUDA - # .cu files don't get these options propagated to nvcc since that would - # cause the build to fail. - set_property( - TARGET pybind11::windows_extras - APPEND - PROPERTY INTERFACE_COMPILE_OPTIONS $<$<NOT:$<CONFIG:Debug>>:$<$<COMPILE_LANGUAGE:CXX>:/MP>>) + # /MP enables multithreaded builds (relevant when there are many files) for MSVC + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") # no Clang no Intel + if(CMAKE_VERSION VERSION_LESS 3.11) + set_property( + TARGET pybind11::windows_extras + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS $<$<NOT:$<CONFIG:Debug>>:/MP>) + else() + # Only set these options for C++ files. This is important so that, for + # instance, projects that include other types of source files like CUDA + # .cu files don't get these options propagated to nvcc since that would + # cause the build to fail. + set_property( + TARGET pybind11::windows_extras + APPEND + PROPERTY INTERFACE_COMPILE_OPTIONS + $<$<NOT:$<CONFIG:Debug>>:$<$<COMPILE_LANGUAGE:CXX>:/MP>>) + endif() endif() endif() @@ -184,11 +163,19 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) elseif( - PYBIND11_FINDPYTHON + _pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON OR Python_FOUND OR Python2_FOUND OR Python3_FOUND) @@ -298,27 +285,50 @@ function(_pybind11_return_if_cxx_and_linker_flags_work result cxxflags linkerfla endfunction() function(_pybind11_generate_lto target prefer_thin_lto) + if(MINGW) + message(STATUS "${target} disabled (problems with undefined symbols for MinGW for now)") + return() + endif() + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") set(cxx_append "") set(linker_append "") if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT APPLE) # Clang Gold plugin does not support -Os; append -O3 to MinSizeRel builds to override it set(linker_append ";$<$<CONFIG:MinSizeRel>:-O3>") - elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND NOT MINGW) set(cxx_append ";-fno-fat-lto-objects") endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND prefer_thin_lto) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + set(NO_FLTO_ARCH TRUE) + else() + set(NO_FLTO_ARCH FALSE) + endif() + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" + AND prefer_thin_lto + AND NOT NO_FLTO_ARCH) _pybind11_return_if_cxx_and_linker_flags_work( HAS_FLTO_THIN "-flto=thin${cxx_append}" "-flto=thin${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() - if(NOT HAS_FLTO_THIN) + if(NOT HAS_FLTO_THIN AND NOT NO_FLTO_ARCH) _pybind11_return_if_cxx_and_linker_flags_work( HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") + # IntelLLVM equivalent to LTO is called IPO; also IntelLLVM is WIN32/UNIX + # WARNING/HELP WANTED: This block of code is currently not covered by pybind11 GitHub Actions! + if(WIN32) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-Qipo" "-Qipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + else() + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") # Intel equivalent to LTO is called IPO _pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO "-ipo" "-ipo" @@ -370,11 +380,13 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- diff --git a/tools/pybind11Config.cmake.in b/tools/pybind11Config.cmake.in index 9921aeb3..5734f437 100644 --- a/tools/pybind11Config.cmake.in +++ b/tools/pybind11Config.cmake.in @@ -13,7 +13,7 @@ This module sets the following variables in your project: ``pybind11_VERSION`` pybind11 version in format Major.Minor.Release ``pybind11_VERSION_TYPE`` - pybind11 version type (dev, release) + pybind11 version type (``dev*`` or empty for a release) ``pybind11_INCLUDE_DIRS`` Directories where pybind11 and python headers are located. ``pybind11_INCLUDE_DIR`` @@ -51,8 +51,6 @@ complex applications, and they are available in all modes: Python headers too. ``pybind11::python_link_helper`` Just the "linking" part of ``pybind11:module``, for CMake < 3.15. -``pybind11::python2_no_register`` - Quiets the warning/error when mixing C++14+ and Python 2, also included in ``pybind11::module``. ``pybind11::thin_lto`` An alternative to ``INTERPROCEDURAL_OPTIMIZATION``. ``pybind11::lto`` @@ -65,7 +63,9 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). New FindPython mode ^^^^^^^^^^^^^^^^^^^ @@ -88,7 +88,7 @@ you can either use the basic targets, or use the FindPython tools: set_target_properties(MyModule2 PROPERTIES INTERPROCEDURAL_OPTIMIZATION ON CXX_VISIBILITY_PRESET ON - VISIBLITY_INLINES_HIDDEN ON) + VISIBILITY_INLINES_HIDDEN ON) If you build targets yourself, you may be interested in stripping the output for reduced size; this is the one other feature that the helper function gives you. @@ -139,7 +139,7 @@ This module defines the following commands to assist with creating Python module pybind11_add_module(<target> [STATIC|SHARED|MODULE] - [THIN_LTO] [OPT_SIZE] [NO_EXTRAS] [WITHOUT_SOBAI] + [THIN_LTO] [OPT_SIZE] [NO_EXTRAS] [WITHOUT_SOABI] <files>... ) @@ -147,7 +147,7 @@ Add a module and setup all helpers. You can select the type of the library; the default is ``MODULE``. There are several options: ``OPT_SIZE`` - Optimize for size, even if the ``CMAKE_BUILD_TYPE`` is not ``RelSize``. + Optimize for size, even if the ``CMAKE_BUILD_TYPE`` is not ``MinSizeRel``. ``THIN_LTO`` Use thin TLO instead of regular if there's a choice (pybind11's selection is disabled if ``CMAKE_INTERPROCEDURAL_OPTIMIZATIONS`` is set). @@ -195,13 +195,14 @@ Using ``find_package`` with version info is not recommended except for release v .. code-block:: cmake find_package(pybind11 CONFIG) - find_package(pybind11 2.0 EXACT CONFIG REQUIRED) + find_package(pybind11 2.9 EXACT CONFIG REQUIRED) #]=============================================================================] @PACKAGE_INIT@ # Location of pybind11/pybind11.h -set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/@CMAKE_INSTALL_INCLUDEDIR@") +# This will be relative unless explicitly set as absolute +set(pybind11_INCLUDE_DIR "@pybind11_INCLUDEDIR@") set(pybind11_LIBRARY "") set(pybind11_DEFINITIONS USING_pybind11) @@ -227,6 +228,6 @@ include("${CMAKE_CURRENT_LIST_DIR}/pybind11Common.cmake") if(NOT pybind11_FIND_QUIETLY) message( STATUS - "Found pybind11: ${pybind11_INCLUDE_DIR} (found version \"${pybind11_VERSION}\" ${pybind11_VERSION_TYPE})" + "Found pybind11: ${pybind11_INCLUDE_DIR} (found version \"${pybind11_VERSION}${pybind11_VERSION_TYPE}\")" ) endif() diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 18da8be1..7d7424a7 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -5,6 +5,12 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. +if(CMAKE_VERSION VERSION_LESS 3.12) + message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") +endif() + +include_guard(DIRECTORY) + get_property( is_config TARGET pybind11::headers @@ -16,13 +22,7 @@ else() set(_pybind11_quiet "") endif() -if(CMAKE_VERSION VERSION_LESS 3.12) - message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") -endif() - -if(NOT Python_FOUND - AND NOT Python3_FOUND - AND NOT Python2_FOUND) +if(NOT Python_FOUND AND NOT Python3_FOUND) if(NOT DEFINED Python_FIND_IMPLEMENTATIONS) set(Python_FIND_IMPLEMENTATIONS CPython PyPy) endif() @@ -32,7 +32,7 @@ if(NOT Python_FOUND set(Python_ROOT_DIR "$ENV{pythonLocation}") endif() - find_package(Python REQUIRED COMPONENTS Interpreter Development ${_pybind11_quiet}) + find_package(Python 3.6 REQUIRED COMPONENTS Interpreter Development ${_pybind11_quiet}) # If we are in submodule mode, export the Python targets to global targets. # If this behavior is not desired, FindPython _before_ pybind11. @@ -49,19 +49,10 @@ if(Python_FOUND) set(_Python Python CACHE INTERNAL "" FORCE) -elseif(Python3_FOUND AND NOT Python2_FOUND) +elseif(Python3_FOUND) set(_Python Python3 CACHE INTERNAL "" FORCE) -elseif(Python2_FOUND AND NOT Python3_FOUND) - set(_Python - Python2 - CACHE INTERNAL "" FORCE) -else() - message(AUTHOR_WARNING "Python2 and Python3 both present, pybind11 in " - "PYBIND11_NOPYTHON mode (manually activate to silence warning)") - set(_pybind11_nopython ON) - return() endif() if(PYBIND11_MASTER_PROJECT) @@ -82,6 +73,15 @@ if(NOT DEFINED ${_Python}_EXECUTABLE) endif() +if(NOT ${_Python}_EXECUTABLE STREQUAL PYBIND11_PYTHON_EXECUTABLE_LAST) + # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed + unset(PYTHON_IS_DEBUG CACHE) + unset(PYTHON_MODULE_EXTENSION CACHE) + set(PYBIND11_PYTHON_EXECUTABLE_LAST + "${${_Python}_EXECUTABLE}" + CACHE INTERNAL "Python executable during the last CMake run") +endif() + if(NOT DEFINED PYTHON_IS_DEBUG) # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter execute_process( @@ -99,7 +99,7 @@ if(NOT DEFINED PYTHON_MODULE_EXTENSION) execute_process( COMMAND "${${_Python}_EXECUTABLE}" "-c" - "from distutils import sysconfig as s;print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" + "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR OUTPUT_STRIP_TRAILING_WHITESPACE) @@ -126,7 +126,7 @@ if(PYTHON_IS_DEBUG) PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG) endif() -# Check on every access - since Python2 and Python3 could have been used - do nothing in that case. +# Check on every access - since Python can change - do nothing in that case. if(DEFINED ${_Python}_INCLUDE_DIRS) # Only add Python for build - must be added during the import for config @@ -148,13 +148,6 @@ if(DEFINED ${_Python}_INCLUDE_DIRS) CACHE INTERNAL "Directories where pybind11 and possibly Python headers are located") endif() -if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS 3) - set_property( - TARGET pybind11::pybind11 - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES pybind11::python2_no_register) -endif() - # In CMake 3.18+, you can find these separately, so include an if if(TARGET ${_Python}::Python) set_property( @@ -194,8 +187,6 @@ function(pybind11_add_module target_name) python_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") python3_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) - elseif("${_Python}" STREQUAL "Python2") - python2_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) else() message(FATAL_ERROR "Cannot detect FindPython version: ${_Python}") endif() @@ -212,10 +203,6 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::windows_extras) endif() - if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS 3) - target_link_libraries(${target_name} PRIVATE pybind11::python2_no_register) - endif() - # -fvisibility=hidden is required to allow multiple modules compiled against # different pybind versions to work properly, and for some features (e.g. # py::module_local). We force it on everything inside the `pybind11` @@ -230,7 +217,7 @@ function(pybind11_add_module target_name) endif() # If we don't pass a WITH_SOABI or WITHOUT_SOABI, use our own default handling of extensions - if(NOT ARG_WITHOUT_SOABI OR NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS) + if(NOT ARG_WITHOUT_SOABI AND NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS) pybind11_extension(${target_name}) endif() @@ -246,7 +233,9 @@ function(pybind11_add_module target_name) endif() endif() - if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Use case-insensitive comparison to match the result of $<CONFIG:cfgs> + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 32313539..66ad00a4 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -5,6 +5,11 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. +# include_guard(global) (pre-CMake 3.10) +if(TARGET pybind11::python_headers) + return() +endif() + # Built-in in CMake 3.5+ include(CMakeParseArguments) @@ -38,38 +43,32 @@ endif() # A user can set versions manually too set(Python_ADDITIONAL_VERSIONS - "3.10;3.9;3.8;3.7;3.6;3.5;3.4" + "3.11;3.10;3.9;3.8;3.7;3.6" CACHE INTERNAL "") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") find_package(PythonLibsNew ${PYBIND11_PYTHON_VERSION} MODULE REQUIRED ${_pybind11_quiet}) list(REMOVE_AT CMAKE_MODULE_PATH -1) +# Makes a normal variable a cached variable +macro(_PYBIND11_PROMOTE_TO_CACHE NAME) + set(_tmp_ptc "${${NAME}}") + # CMake 3.21 complains if a cached variable is shadowed by a normal one + unset(${NAME}) + set(${NAME} + "${_tmp_ptc}" + CACHE INTERNAL "") +endmacro() + # Cache variables so pybind11_add_module can be used in parent projects -set(PYTHON_INCLUDE_DIRS - ${PYTHON_INCLUDE_DIRS} - CACHE INTERNAL "") -set(PYTHON_LIBRARIES - ${PYTHON_LIBRARIES} - CACHE INTERNAL "") -set(PYTHON_MODULE_PREFIX - ${PYTHON_MODULE_PREFIX} - CACHE INTERNAL "") -set(PYTHON_MODULE_EXTENSION - ${PYTHON_MODULE_EXTENSION} - CACHE INTERNAL "") -set(PYTHON_VERSION_MAJOR - ${PYTHON_VERSION_MAJOR} - CACHE INTERNAL "") -set(PYTHON_VERSION_MINOR - ${PYTHON_VERSION_MINOR} - CACHE INTERNAL "") -set(PYTHON_VERSION - ${PYTHON_VERSION} - CACHE INTERNAL "") -set(PYTHON_IS_DEBUG - "${PYTHON_IS_DEBUG}" - CACHE INTERNAL "") +_pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS) +_pybind11_promote_to_cache(PYTHON_LIBRARIES) +_pybind11_promote_to_cache(PYTHON_MODULE_PREFIX) +_pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION) +_pybind11_promote_to_cache(PYTHON_VERSION_MAJOR) +_pybind11_promote_to_cache(PYTHON_VERSION_MINOR) +_pybind11_promote_to_cache(PYTHON_VERSION) +_pybind11_promote_to_cache(PYTHON_IS_DEBUG) if(PYBIND11_MASTER_PROJECT) if(PYTHON_MODULE_EXTENSION MATCHES "pypy") @@ -116,25 +115,37 @@ if(PYTHON_IS_DEBUG) PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG) endif() -set_property( - TARGET pybind11::module - APPEND - PROPERTY - INTERFACE_LINK_LIBRARIES pybind11::python_link_helper - "$<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:Cygwin>>:$<BUILD_INTERFACE:${PYTHON_LIBRARIES}>>") +# The <3.11 code here does not support release/debug builds at the same time, like on vcpkg +if(CMAKE_VERSION VERSION_LESS 3.11) + set_property( + TARGET pybind11::module + APPEND + PROPERTY + INTERFACE_LINK_LIBRARIES + pybind11::python_link_helper + "$<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:Cygwin>>:$<BUILD_INTERFACE:${PYTHON_LIBRARIES}>>" + ) -if(PYTHON_VERSION VERSION_LESS 3) set_property( - TARGET pybind11::pybind11 + TARGET pybind11::embed APPEND - PROPERTY INTERFACE_LINK_LIBRARIES pybind11::python2_no_register) + PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $<BUILD_INTERFACE:${PYTHON_LIBRARIES}>) +else() + # The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside + # of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are + # target_link_library keywords rather than real libraries. + add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) + target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) + target_link_libraries( + pybind11::module + INTERFACE + pybind11::python_link_helper + "$<$<OR:$<PLATFORM_ID:Windows>,$<PLATFORM_ID:Cygwin>>:pybind11::_ClassicPythonLibraries>") + + target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 + pybind11::_ClassicPythonLibraries) endif() -set_property( - TARGET pybind11::embed - APPEND - PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $<BUILD_INTERFACE:${PYTHON_LIBRARIES}>) - function(pybind11_extension name) # The prefix and extension are provided by FindPythonLibsNew.cmake set_target_properties(${name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" @@ -201,7 +212,9 @@ function(pybind11_add_module target_name) endif() endif() - if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Use case-insensitive comparison to match the result of $<CONFIG:cfgs> + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) pybind11_strip(${target_name}) endif() diff --git a/tools/setup_global.py.in b/tools/setup_global.py.in index 4cf040b2..885ac5c7 100644 --- a/tools/setup_global.py.in +++ b/tools/setup_global.py.in @@ -1,17 +1,11 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # Setup script for pybind11-global (in the sdist or in tools/setup_global.py in the repository) # This package is targeted for easy use from CMake. -import contextlib import glob import os import re -import shutil -import subprocess -import sys -import tempfile # Setuptools has to be before distutils from setuptools import setup @@ -33,8 +27,11 @@ class InstallHeadersNested(install_headers): main_headers = glob.glob("pybind11/include/pybind11/*.h") detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") +eigen_headers = glob.glob("pybind11/include/pybind11/eigen/*.h") +stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") -headers = main_headers + detail_headers +pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") +headers = main_headers + detail_headers + stl_headers + eigen_headers cmdclass = {"install_headers": InstallHeadersNested} $extra_cmd @@ -56,8 +53,11 @@ setup( headers=headers, data_files=[ (base + "share/cmake/pybind11", cmake_files), + (base + "share/pkgconfig", pkgconfig_files), (base + "include/pybind11", main_headers), (base + "include/pybind11/detail", detail_headers), + (base + "include/pybind11/eigen", eigen_headers), + (base + "include/pybind11/stl", stl_headers), ], cmdclass=cmdclass, ) diff --git a/tools/setup_main.py.in b/tools/setup_main.py.in index 2231a08f..6358cc7b 100644 --- a/tools/setup_main.py.in +++ b/tools/setup_main.py.in @@ -1,5 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- +#!/usr/bin/env python3 # Setup script (in the sdist or in tools/setup_main.py in the repository) @@ -16,13 +15,19 @@ setup( "pybind11", "pybind11.include.pybind11", "pybind11.include.pybind11.detail", + "pybind11.include.pybind11.eigen", + "pybind11.include.pybind11.stl", "pybind11.share.cmake.pybind11", + "pybind11.share.pkgconfig", ], package_data={ - "pybind11": ["py.typed", "*.pyi"], + "pybind11": ["py.typed"], "pybind11.include.pybind11": ["*.h"], "pybind11.include.pybind11.detail": ["*.h"], + "pybind11.include.pybind11.eigen": ["*.h"], + "pybind11.include.pybind11.stl": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], + "pybind11.share.pkgconfig": ["*.pc"], }, extras_require={ "global": ["pybind11_global==$version"] @@ -30,6 +35,9 @@ setup( entry_points={ "console_scripts": [ "pybind11-config = pybind11.__main__:main", + ], + "pipx.run": [ + "pybind11 = pybind11.__main__:main", ] }, cmdclass=cmdclass |
