Implement rustdoc-scrape-examples

.cargo/config

@@ -1,2 +1,8 @@
 [alias]
 xtask = "run --package xtask --"
+pyo3_doc = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend"
+pyo3_doc_scrape = "doc --lib --no-default-features --features=full --no-deps --workspace --open --exclude pyo3-macros --exclude pyo3-macros-backend -Z unstable-options -Z rustdoc-scrape-examples=examples"
+pyo3_doc_internal = "doc --lib --no-default-features --features=full --no-deps --workspace --open --document-private-items -Z unstable-options -Z rustdoc-scrape-examples=examples"
+
+[build]
+rustdocflags = ["--cfg", "docsrs"]

.github/workflows/ci.yml

@@ -64,8 +64,8 @@ jobs:
             echo "suppress_build_script_link_lines=true" >> config.txt
             PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features
             PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features --features "abi3"
-            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features --features "$(make list_all_additive_features)"
-            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --no-default-features --features "abi3 $(make list_all_additive_features)"
+            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --features full
+            PYO3_BUILD_CONFIG=$(pwd)/config.txt cargo check --all-targets --features "abi3 full"
           done
 
   build:
@@ -146,20 +146,14 @@ jobs:
         name: Prepare LD_LIBRARY_PATH (Ubuntu only)
         run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV
 
-      - name: Prepare workflow settings
-        id: settings
-        shell: bash
-        run: |
-          echo "::set-output name=all_additive_features::$(make list_all_additive_features)"
-
       - if: matrix.msrv == 'MSRV'
         name: Prepare minimal package versions (MSRV only)
         run: |
           cargo update -p indexmap --precise 1.6.2
           cargo update -p hashbrown:0.11.2 --precise 0.9.1
 
       - name: Build docs
-        run: cargo doc --no-deps --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo doc --no-deps --no-default-features --features full
 
       - name: Build (no features)
         run: cargo build --lib --tests --no-default-features
@@ -182,26 +176,26 @@ jobs:
           cargo test --no-default-features
 
       - name: Build (all additive features)
-        run: cargo build --lib --tests --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo build --lib --tests --no-default-features --features full
 
       - if: ${{ startsWith(matrix.python-version, 'pypy') }}
         name: Build PyPy (abi3-py37)
-        run: cargo build --lib --tests --no-default-features --features "abi3-py37 ${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo build --lib --tests --no-default-features --features "abi3-py37 full"
 
       # Run tests (except on PyPy, because no embedding API).
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test
-        run: cargo test --no-default-features --features "${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo test --no-default-features --features full
 
       # Run tests again, but in abi3 mode
       - if: ${{ !startsWith(matrix.python-version, 'pypy') }}
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3 ${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo test --no-default-features --features "abi3 full"
 
       # Run tests again, for abi3-py37 (the minimal Python version)
       - if: ${{ (!startsWith(matrix.python-version, 'pypy')) && (matrix.python-version != '3.7') }}
         name: Test (abi3-py37)
-        run: cargo test --no-default-features --features "abi3-py37 ${{ steps.settings.outputs.all_additive_features }}"
+        run: cargo test --no-default-features --features "abi3-py37 full"
 
       - name: Test proc-macro code
         run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml
@@ -281,8 +275,8 @@ jobs:
           cargo llvm-cov clean --workspace
           cargo llvm-cov --package $ALL_PACKAGES --no-report
           cargo llvm-cov --package $ALL_PACKAGES --no-report --features abi3
-          cargo llvm-cov --package $ALL_PACKAGES --no-report --features $(make list_all_additive_features)
-          cargo llvm-cov --package $ALL_PACKAGES --no-report --features abi3 $(make list_all_additive_features)
+          cargo llvm-cov --package $ALL_PACKAGES --no-report --features full
+          cargo llvm-cov --package $ALL_PACKAGES --no-report --features "abi3 full"
           cargo llvm-cov --package $ALL_PACKAGES --no-run --lcov --output-path coverage.lcov
         shell: bash
         env:

.github/workflows/guide.yml

@@ -35,6 +35,29 @@ jobs:
           TAG_NAME="${GITHUB_REF##*/}"
           echo "::set-output name=tag_name::${TAG_NAME}"
 
+      # Build some internal docs and inject a banner on top of it.
+      - name: Build the internal docs
+        run: |
+          mkdir target
+          mkdir -p gh-pages-build/internal
+          echo "<div class='internal-banner' style='position:fixed; z-index: 99999; color:red;border:3px solid red;margin-left: auto; margin-right: auto; width: 430px;left:0;right: 0;'><div style='display: flex; align-items: center; justify-content: center;'> ⚠️ Internal Docs ⚠️ Not Public API 👉 <a href='https://pyo3.rs/main/doc/pyo3/index.html' style='color:red;text-decoration:underline;'>Official Docs Here</a></div></div>" > target/banner.html
+          cargo +nightly pyo3_doc_internal
+          cp -r target/doc gh-pages-build/internal
+        env:
+          RUSTDOCFLAGS: "--cfg docsrs --Z unstable-options --document-hidden-items --html-before-content target/banner.html"
+
+      - name: Deploy internal docs
+        if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }}
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./gh-pages-build/internal
+          destination_dir: internal
+          full_commit_message: "Upload internal documentation"
+
+      - name: Clear the extra artefacts created earlier 
+        run: rm -rf target
+
       # This builds the book in gh-pages-build. See https://github.com/rust-lang-nursery/mdBook/issues/698
       - name: Build the guide
         run: mdbook build -d ../gh-pages-build guide
@@ -44,19 +67,18 @@ jobs:
         # This adds the docs to gh-pages-build/doc
       - name: Build the doc
         run: |
-          cargo +nightly rustdoc --lib --no-default-features --features="macros num-bigint num-complex hashbrown indexmap serde multiple-pymethods eyre anyhow" -- --cfg docsrs
+          cargo +nightly pyo3_doc_scrape
           cp -r target/doc gh-pages-build/doc
           echo "<meta http-equiv=refresh content=0;url=pyo3/index.html>" > gh-pages-build/doc/index.html
 
-      - name: Deploy
+      - name: Deploy docs and the guide
         if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'release' }}
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./gh-pages-build/
           destination_dir: ${{ steps.prepare_tag.outputs.tag_name }}
           full_commit_message: "Upload documentation for ${{ steps.prepare_tag.outputs.tag_name }}"
-
   release:
     needs: build
     runs-on: ubuntu-latest

Cargo.toml

@@ -79,6 +79,10 @@ auto-initialize = []
 # Optimizes PyObject to Vec conversion and so on.
 nightly = []
 
+# Activates all additional features
+# This is mostly intended for testing purposes - activating *all* of these isn't particularly useful.
+full = ["macros", "multiple-pymethods", "num-bigint", "num-complex", "hashbrown", "serde", "indexmap", "eyre", "anyhow"]
+
 [[bench]]
 name = "bench_call"
 harness = false

Contributing.md

@@ -9,7 +9,9 @@ If you want to become familiar with the codebase, see
 
 ## Getting started contributing
 
-Please join in with any part of PyO3 which interests you. We use GitHub issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it.
+Please join in with any part of PyO3 which interests you. We use GitHub issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it. 
+
+You can browse the API of the non-public parts of PyO3 [here](https://pyo3.rs/internal/doc/pyo3/index.html).
 
 The following sections also contain specific ideas on where to start contributing to PyO3.
 
@@ -34,7 +36,8 @@ There are some specific areas of focus where help is currently needed for the do
 - Issues requesting documentation improvements are tracked with the [documentation](https://github.com/PyO3/pyo3/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) label.
 - Not all APIs had docs or examples when they were made. The goal is to have documentation on all PyO3 APIs ([#306](https://github.com/PyO3/pyo3/issues/306)). If you see an API lacking a doc, please write one and open a PR!
 
-You can build the docs (including all features) with `cargo +nightly rustdoc --features="$(make list_all_additive_features)" --open -- --cfg docsrs`.
+You can build the docs (including all features) with 
+```cargo +nightly pyo3_doc_scrape```
 
 #### Doctests
 
@@ -47,7 +50,7 @@ https://doc.rust-lang.org/rustdoc/documentation-tests.html for a guide on doctes
 You can preview the user guide by building it locally with `mdbook`.
 
 First, [install `mdbook`](https://rust-lang.github.io/mdBook/cli/index.html). Then, run
-`mdbook build -d ../gh-pages-build guide --open`.
+```mdbook build -d ../gh-pages-build guide --open```
 
 ### Help design the next PyO3
 
@@ -71,7 +74,7 @@ Formatting, linting and tests are checked for all Rust and Python code. In addit
 
 Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version.
 
-If you are adding a new feature, you should add it to the `ALL_ADDITIVE_FEATURES` declaration in the `Makefile` so that it is tested in CI.
+If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI.
 
 ## Python and Rust version support policy
 

examples/decorator/tests/example.py

@@ -0,0 +1,11 @@
+@Counter
+def say_hello():
+    print("hello")
+
+
+say_hello()
+say_hello()
+say_hello()
+say_hello()
+
+assert say_hello.count == 4

guide/src/class/call.md

@@ -21,7 +21,7 @@ An example crate containing this pyclass can be found [here](https://github.com/
 Python code:
 
 ```python
-{{#include ../../../examples/decorator/src/test.py}}
+{{#include ../../../examples/decorator/tests/example.py}}
 ```
 
 Output:

pyo3-macros-backend/src/lib.rs

@@ -1,7 +1,7 @@
 // Copyright (c) 2017-present PyO3 Project and Contributors
 //! This crate contains the implementation of the proc macro attributes
 
-#![cfg_attr(docsrs, feature(doc_cfg))]
+#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
 #![recursion_limit = "1024"]
 
 // Listed first so that macros in this module are available in the rest of the crate.

pyo3-macros-backend/src/method.rs

@@ -64,17 +64,17 @@ impl<'a> FnArg<'a> {
 
 #[derive(Clone, PartialEq, Debug, Copy, Eq)]
 pub enum MethodTypeAttribute {
-    /// #[new]
+    /// `#[new]`
     New,
-    /// #[classmethod]
+    /// `#[classmethod]`
     ClassMethod,
-    /// #[classattr]
+    /// `#[classattr]`
     ClassAttribute,
-    /// #[staticmethod]
+    /// `#[staticmethod]`
     StaticMethod,
-    /// #[getter]
+    /// `#[getter]`
     Getter,
-    /// #[setter]
+    /// `#[setter]`
     Setter,
 }
 

pyo3-macros/src/lib.rs

@@ -2,7 +2,7 @@
 //! This crate declares only the proc macro attributes, as a crate defining proc macro attributes
 //! must not contain any other public items.
 
-#![cfg_attr(docsrs, feature(doc_cfg))]
+#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))]
 extern crate proc_macro;
 
 use proc_macro::TokenStream;

src/conversions/anyhow.rs

@@ -1,6 +1,9 @@
 #![cfg(feature = "anyhow")]
 
-//! A conversion from [anyhow]’s [`Error`][anyhow_error] type to [`PyErr`].
+//! A conversion from
+//! [anyhow](https://docs.rs/anyhow/ "A trait object based error system for easy idiomatic error handling in Rust applications.")’s
+//! [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html "Anyhows `Error` type, a wrapper around a dynamic error type")
+//! type to [`PyErr`].
 //!
 //! Use of an error handling library like [anyhow] is common in application code and when you just
 //! want error handling to be easy. If you are writing a library or you need more control over your
@@ -101,8 +104,6 @@
 //! }
 //! ```
 //!
-//! [anyhow]: https://docs.rs/anyhow/ "A trait object based error system for easy idiomatic error handling in Rust applications."
-//! [anyhow_error]: https://docs.rs/anyhow/latest/anyhow/struct.Error.html "Anyhows `Error` type, a wrapper around a dynamic error type"
 //! [`RuntimeError`]: https://docs.python.org/3/library/exceptions.html#RuntimeError "Built-in Exceptions — Python documentation"
 //! [Error handling]: https://doc.rust-lang.org/book/ch09-02-recoverable-errors-with-result.html "Recoverable Errors with Result - The Rust Programming Language"
 

src/conversions/eyre.rs

@@ -1,6 +1,8 @@
 #![cfg(feature = "eyre")]
 
-//! A conversion from [eyre]’s [`Report`] type to [`PyErr`].
+//! A conversion from
+//! [eyre](https://docs.rs/eyre/ "A library for easy idiomatic error handling and reporting in Rust applications.")’s
+//! [`Report`] type to [`PyErr`].
 //!
 //! Use of an error handling library like [eyre] is common in application code and when you just
 //! want error handling to be easy. If you are writing a library or you need more control over your

src/impl_/not_send.rs

@@ -3,7 +3,7 @@ use std::marker::PhantomData;
 use crate::Python;
 
 /// A marker type that makes the type !Send.
-/// Workaround for lack of !Send on stable (https://github.com/rust-lang/rust/issues/68318).
+/// Workaround for lack of !Send on stable (<https://github.com/rust-lang/rust/issues/68318>).
 pub(crate) struct NotSend(PhantomData<*mut Python<'static>>);
 
 pub(crate) const NOT_SEND: NotSend = NotSend(PhantomData);