Update docs for raw FFI use (#4665)

* update top-level docs for pyo3-ffi

* update listing of build config variables

* don't depend on a file that doesn't exist

* Move pyo3-ffi examples under pyo3-ffi repo

* apply review comments

* add changelog entry for the moved examples

* fix URL anchor in pyo3-ffi docs

* remove problematic links

examples/README.md

@@ -9,9 +9,11 @@ Below is a brief description of each of these:
 | `decorator` | A project showcasing the example from the [Emulating callable objects](https://pyo3.rs/latest/class/call.html) chapter of the guide. |
 | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. |
 | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. |
-| `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. |
 | `plugin` | Illustrates how to use Python as a scripting language within a Rust application |
-| `sequential` | Illustrates how to use pyo3-ffi to write subinterpreter-safe modules |
+
+Note that there are also other examples in the `pyo3-ffi/examples`
+directory that illustrate how to create rust extensions using raw FFI calls into
+the CPython C API instead of using PyO3's abstractions.
 
 ## Creating new projects from these examples
 

newsfragments/4665.changed.md

@@ -0,0 +1,2 @@
+* The `sequential` and `string-sum` examples have moved into a new `examples`
+  directory in the `pyo3-ffi` crate.

noxfile.py

@@ -65,6 +65,8 @@ def test_py(session: nox.Session) -> None:
     _run(session, "nox", "-f", "pytests/noxfile.py", external=True)
     for example in glob("examples/*/noxfile.py"):
         _run(session, "nox", "-f", example, external=True)
+    for example in glob("pyo3-ffi/examples/*/noxfile.py"):
+        _run(session, "nox", "-f", example, external=True)
 
 
 @nox.session(venv_backend="none")

pyo3-ffi/examples/README.md

@@ -0,0 +1,21 @@
+# `pyo3-ffi` Examples
+
+These example crates are a collection of toy extension modules built with
+`pyo3-ffi`. They are all tested using `nox` in PyO3's CI.
+
+Below is a brief description of each of these:
+
+| Example | Description |
+| `word-count` | Illustrates how to use pyo3-ffi to write a static rust extension |
+| `sequential` | Illustrates how to use pyo3-ffi to write subinterpreter-safe modules using multi-phase module initialization |
+
+## Creating new projects from these examples
+
+To copy an example, use [`cargo-generate`](https://crates.io/crates/cargo-generate). Follow the commands below, replacing `<example>` with the example to start from:
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/<example>
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/sequential/Cargo.toml → pyo3-ffi/examples/sequential/Cargo.toml

@@ -8,6 +8,6 @@ name = "sequential"
 crate-type = ["cdylib", "lib"]
 
 [dependencies]
-pyo3-ffi = { path = "../../pyo3-ffi", features = ["extension-module"] }
+pyo3-ffi = { path = "../../", features = ["extension-module"] }
 
 [workspace]

examples/string-sum/Cargo.toml → pyo3-ffi/examples/string-sum/Cargo.toml

@@ -8,6 +8,6 @@ name = "string_sum"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3-ffi = { path = "../../pyo3-ffi", features = ["extension-module"] }
+pyo3-ffi = { path = "../../", features = ["extension-module"] }
 
 [workspace]

pyo3-ffi/src/lib.rs

@@ -64,8 +64,8 @@
 //! your `Cargo.toml`:
 //!
 //! ```toml
-//! [build-dependency]
-//! pyo3-build-config = "VER"
+//! [build-dependencies]
+#![doc = concat!("pyo3-build-config =\"", env!("CARGO_PKG_VERSION"),  "\"")]
 //! ```
 //!
 //! And then either create a new `build.rs` file in the project root or modify
@@ -108,104 +108,31 @@
 //! [dependencies.pyo3-ffi]
 #![doc = concat!("version = \"", env!("CARGO_PKG_VERSION"),  "\"")]
 //! features = ["extension-module"]
+//!
+//! [build-dependencies]
+//! # This is only necessary if you need to configure your build based on
+//! # the Python version or the compile-time configuration for the interpreter.
+#![doc = concat!("pyo3_build_config = \"", env!("CARGO_PKG_VERSION"),  "\"")]
 //! ```
 //!
-//! **`src/lib.rs`**
-//! ```rust
-//! use std::os::raw::c_char;
-//! use std::ptr;
-//!
-//! use pyo3_ffi::*;
-//!
-//! static mut MODULE_DEF: PyModuleDef = PyModuleDef {
-//!     m_base: PyModuleDef_HEAD_INIT,
-//!     m_name: c_str!("string_sum").as_ptr(),
-//!     m_doc: c_str!("A Python module written in Rust.").as_ptr(),
-//!     m_size: 0,
-//!     m_methods: unsafe { METHODS.as_mut_ptr().cast() },
-//!     m_slots: std::ptr::null_mut(),
-//!     m_traverse: None,
-//!     m_clear: None,
-//!     m_free: None,
-//! };
-//!
-//! static mut METHODS: [PyMethodDef; 2] = [
-//!     PyMethodDef {
-//!         ml_name: c_str!("sum_as_string").as_ptr(),
-//!         ml_meth: PyMethodDefPointer {
-//!             PyCFunctionFast: sum_as_string,
-//!         },
-//!         ml_flags: METH_FASTCALL,
-//!         ml_doc: c_str!("returns the sum of two integers as a string").as_ptr(),
-//!     },
-//!     // A zeroed PyMethodDef to mark the end of the array.
-//!     PyMethodDef::zeroed()
-//! ];
-//!
-//! // The module initialization function, which must be named `PyInit_<your_module>`.
-//! #[allow(non_snake_case)]
-//! #[no_mangle]
-//! pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject {
-//!     PyModule_Create(ptr::addr_of_mut!(MODULE_DEF))
-//! }
+//! If you need to use conditional compilation based on Python version or how
+//! Python was compiled, you need to add `pyo3-build-config` as a
+//! `build-dependency` in your `Cargo.toml` as in the example above and either
+//! create a new `build.rs` file or modify an existing one so that
+//! `pyo3_build_config::use_pyo3_cfgs()` gets called at build time:
 //!
-//! pub unsafe extern "C" fn sum_as_string(
-//!     _self: *mut PyObject,
-//!     args: *mut *mut PyObject,
-//!     nargs: Py_ssize_t,
-//! ) -> *mut PyObject {
-//!     if nargs != 2 {
-//!         PyErr_SetString(
-//!             PyExc_TypeError,
-//!             c_str!("sum_as_string() expected 2 positional arguments").as_ptr(),
-//!         );
-//!         return std::ptr::null_mut();
-//!     }
-//!
-//!     let arg1 = *args;
-//!     if PyLong_Check(arg1) == 0 {
-//!         PyErr_SetString(
-//!             PyExc_TypeError,
-//!             c_str!("sum_as_string() expected an int for positional argument 1").as_ptr(),
-//!         );
-//!         return std::ptr::null_mut();
-//!     }
-//!
-//!     let arg1 = PyLong_AsLong(arg1);
-//!     if !PyErr_Occurred().is_null() {
-//!         return ptr::null_mut();
-//!     }
-//!
-//!     let arg2 = *args.add(1);
-//!     if PyLong_Check(arg2) == 0 {
-//!         PyErr_SetString(
-//!             PyExc_TypeError,
-//!             c_str!("sum_as_string() expected an int for positional argument 2").as_ptr(),
-//!         );
-//!         return std::ptr::null_mut();
-//!     }
-//!
-//!     let arg2 = PyLong_AsLong(arg2);
-//!     if !PyErr_Occurred().is_null() {
-//!         return ptr::null_mut();
-//!     }
-//!
-//!     match arg1.checked_add(arg2) {
-//!         Some(sum) => {
-//!             let string = sum.to_string();
-//!             PyUnicode_FromStringAndSize(string.as_ptr().cast::<c_char>(), string.len() as isize)
-//!         }
-//!         None => {
-//!             PyErr_SetString(
-//!                 PyExc_OverflowError,
-//!                 c_str!("arguments too large to add").as_ptr(),
-//!             );
-//!             std::ptr::null_mut()
-//!         }
-//!     }
+//! **`build.rs`**
+//! ```rust,ignore
+//! fn main() {
+//!     pyo3_build_config::use_pyo3_cfgs()
 //! }
 //! ```
 //!
+//! **`src/lib.rs`**
+//! ```rust
+#![doc = include_str!("../examples/string-sum/src/lib.rs")]
+//! ```
+//!
 //! With those two files in place, now `maturin` needs to be installed. This can be done using
 //! Python's package manager `pip`. First, load up a new Python `virtualenv`, and install `maturin`
 //! into it:
@@ -230,6 +157,12 @@
 //! [manually][manual_builds]. Both offer more flexibility than `maturin` but require further
 //! configuration.
 //!
+//! This example stores the module definition statically and uses the `PyModule_Create` function
+//! in the CPython C API to register the module. This is the "old" style for registering modules
+//! and has the limitation that it cannot support subinterpreters. You can also create a module
+//! using the new multi-phase initialization API that does support subinterpreters. See the
+//! `sequential` project located in the `examples` directory at the root of the `pyo3-ffi` crate
+//! for a worked example of how to this using `pyo3-ffi`.
 //!
 //! # Using Python from Rust
 //!
@@ -255,7 +188,7 @@
 #![doc = concat!("[manual_builds]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution.html#manual-builds \"Manual builds - Building and Distribution - PyO3 user guide\"")]
 //! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions"
 //! [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI"
-#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features Reference - PyO3 user guide\"")]
+#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features eference - PyO3 user guide\"")]
 #![allow(
     missing_docs,
     non_camel_case_types,

src/lib.rs

@@ -123,14 +123,25 @@
 //! - `nightly`: Uses  `#![feature(auto_traits, negative_impls)]` to define [`Ungil`] as an auto trait.
 //
 //! ## `rustc` environment flags
-//!
-//! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions.
-//! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`] crate.
-//!
-//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when
-//!  compiling for a given minimum Python version.
+//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`, `Py_3_11`, `Py_3_12`, `Py_3_13`: Marks code that is
+//!    only enabled when compiling for a given minimum Python version.
 //! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled.
+//! - `Py_GIL_DISABLED`: Marks code that runs only in the free-threaded build of CPython.
 //! - `PyPy` - Marks code enabled when compiling for PyPy.
+//! - `GraalPy` - Marks code enabled when compiling for GraalPy.
+//!
+//! Additionally, you can query for the values `Py_DEBUG`, `Py_REF_DEBUG`,
+//! `Py_TRACE_REFS`, and `COUNT_ALLOCS` from `py_sys_config` to query for the
+//! corresponding C build-time defines. For example, to conditionally define
+//! debug code using `Py_DEBUG`, you could do:
+//!
+//! ```rust,ignore
+//! #[cfg(py_sys_config = "Py_DEBUG")]
+//! println!("only runs if python was compiled with Py_DEBUG")
+//! ```
+//! To use these attributes, add [`pyo3-build-config`] as a build dependency in
+//! your `Cargo.toml` and call `pyo3_build_config::use_pyo3_cfgs()` in a
+//! `build.rs` file.
 //!
 //! # Minimum supported Rust and Python versions
 //!