Added some docs, examples, testing feature separation

Author: Andrew J Westlake

.githooks/pre-commit

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cargo check --all-targets
+cargo check --all-targets --features testing

.githooks/pre-push

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-cargo test
+cargo test --features testing

.github/workflows/ci.yml

@@ -85,28 +85,28 @@ jobs:
         run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV
 
       - name: Build docs
-        run: cargo doc --no-default-features --verbose --target ${{ matrix.platform.rust-target }}
+        run: cargo doc --no-default-features --features testing --verbose --target ${{ matrix.platform.rust-target }}
 
       - name: Build (no features)
         run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}
 
       - name: Build (all additive features)
-        run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}
+        run: cargo build --no-default-features --features testing --verbose --target ${{ matrix.platform.rust-target }}
 
       # Run tests (except on PyPy, because no embedding API).
       - if: matrix.python-version != 'pypy-3.6'
         name: Test
-        run: cargo test --no-default-features --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features testing --target ${{ matrix.platform.rust-target }}
 
       # Run tests again, but in abi3 mode
       - if: matrix.python-version != 'pypy-3.6'
         name: Test (abi3)
-        run: cargo test --no-default-features --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features testing --target ${{ matrix.platform.rust-target }}
 
       # Run tests again, for abi3-py36 (the minimal Python version)
       - if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6')
         name: Test (abi3-py36)
-        run: cargo test --no-default-features --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features testing --target ${{ matrix.platform.rust-target }}
 
       - name: Install python test dependencies
         run: |

.github/workflows/guide.yml

@@ -21,7 +21,7 @@ jobs:
         # This adds the docs to gh-pages-build/doc
       - name: Build the doc
         run: |
-          cargo doc --no-deps
+          cargo doc --no-deps --features testing
           mkdir -p gh-pages-build
           cp -r target/doc gh-pages-build/doc
           echo "<meta http-equiv=refresh content=0;url=pyo3/index.html>" > gh-pages-build/doc/index.html

Cargo.toml

@@ -6,13 +6,24 @@ edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+testing = ["clap"]
+default = []
+
 [[test]]
 name = "test_asyncio"
 path = "pytests/test_asyncio.rs"
 harness = false
+required-features = ["testing"]
+
+[[test]]
+name = "test_run_forever"
+path = "pytests/test_run_forever.rs"
+harness = false
+required-features = ["testing"]
 
 [dependencies]
-clap = "2.33"
+clap = { version = "2.33", optional = true }
 futures = "0.3"
 lazy_static = "1.4"
 once_cell = "1.5"

README.md

@@ -7,7 +7,7 @@
 
 [Rust](http://www.rust-lang.org/) bindings for [Python](https://www.python.org/). This includes running and interacting with Python code from a Rust binary, as well as writing native Python modules.
 
-* API Documentation: [stable](https://docs.rs/pyo3-asyncio/)
+* API Documentation: [stable](https://docs.rs/pyo3-asyncio/) | [master](https://awestlake87.github.io/pyo3-asyncio/master/doc)
 
 * Contributing Notes: [github](https://github.com/awestlake87/pyo3-asyncio/blob/master/Contributing.md)
 
@@ -33,13 +33,4 @@ to propagate through the Python layers. This allows panics in tests to exit the
 event loop like before, but may cause unexpected behaviour if a Rust future
 awaits a Python coroutine that awaits a Rust future that panics.
 
-> These scenarios are currently untested in this crate.
-
-## Running the Test
-
-You can run the example test the same way you'd run any other Cargo integration
-test.
-
-```
-$ cargo test
-```
+> These scenarios are currently untested in this crate.

pytests/test_asyncio.rs

@@ -3,15 +3,7 @@ use std::{future::Future, thread, time::Duration};
 use futures::stream::{self};
 use pyo3::prelude::*;
 
-use pyo3_asyncio::test::{parse_args, test_harness, Test};
-
-fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ {
-    move |e| {
-        // We can't display Python exceptions via std::fmt::Display,
-        // so print the error here manually.
-        e.print_and_set_sys_last_vars(py);
-    }
-}
+use pyo3_asyncio::testing::{test_main, Test};
 
 fn test_async_sleep<'p>(
     py: Python<'p>,
@@ -40,30 +32,21 @@ fn test_blocking_sleep() {
     println!("success");
 }
 
-fn py_main(py: Python) -> PyResult<()> {
-    let args = parse_args("Pyo3 Asyncio Test Suite");
-
-    pyo3_asyncio::try_init(py)?;
-
-    pyo3_asyncio::run_until_complete(
-        py,
-        test_harness(
-            stream::iter(vec![
-                Test::new_async("test_async_sleep".into(), test_async_sleep(py)?),
-                Test::new_sync("test_blocking_sleep".into(), || {
-                    test_blocking_sleep();
-                    Ok(())
-                }),
-            ]),
-            args,
-        ),
-    )?;
-
-    Ok(())
-}
-
 fn main() {
-    Python::with_gil(|py| {
-        py_main(py).map_err(dump_err(py)).unwrap();
-    });
+    test_main(stream::iter(vec![
+        Test::new_async(
+            "test_async_sleep".into(),
+            Python::with_gil(|py| {
+                test_async_sleep(py)
+                    .map_err(|e| {
+                        e.print_and_set_sys_last_vars(py);
+                    })
+                    .unwrap()
+            }),
+        ),
+        Test::new_sync("test_blocking_sleep".into(), || {
+            test_blocking_sleep();
+            Ok(())
+        }),
+    ]))
 }

pytests/test_run_forever.rs

@@ -0,0 +1,44 @@
+use std::time::Duration;
+
+use pyo3::prelude::*;
+
+fn dump_err(py: Python<'_>) -> impl FnOnce(PyErr) + '_ {
+    move |e| {
+        // We can't display Python exceptions via std::fmt::Display,
+        // so print the error here manually.
+        e.print_and_set_sys_last_vars(py);
+    }
+}
+
+fn main() {
+    Python::with_gil(|py| {
+        pyo3_asyncio::with_runtime(py, || {
+            pyo3_asyncio::spawn(async move {
+                tokio::time::sleep(Duration::from_secs(1)).await;
+
+                Python::with_gil(|py| {
+                    let event_loop = pyo3_asyncio::get_event_loop();
+
+                    event_loop
+                        .call_method1(
+                            py,
+                            "call_soon_threadsafe",
+                            (event_loop
+                                .getattr(py, "stop")
+                                .map_err(dump_err(py))
+                                .unwrap(),),
+                        )
+                        .map_err(dump_err(py))
+                        .unwrap();
+                })
+            });
+
+            pyo3_asyncio::run_forever(py)?;
+
+            println!("test test_run_forever ... ok");
+            Ok(())
+        })
+        .map_err(dump_err(py))
+        .unwrap();
+    })
+}

src/lib.rs

@@ -1,4 +1,62 @@
-pub mod test;
+#![feature(doc_cfg)]
+#![warn(missing_docs)]
+
+//! Rust Bindings to the Python Asyncio Event Loop
+//!
+//! # Motivation
+//!
+//! This crate aims to provide a convenient interface to manage the interop between Python and
+//! Rust's async/await models. It supports conversions between Rust and Python futures and manages
+//! the event loops for both languages. Python's threading model and GIL can make this interop a bit
+//! trickier than one might expect, so there are a few caveats that users should be aware of.
+//!
+//! ## Why Two Event Loops
+//!
+//! Currently, we don't have a way to run Rust futures directly on Python's event loop. Likewise,
+//! Python's coroutines cannot be directly spawned on a Rust event loop. The two coroutine models
+//! require some additional assistance from their event loops, so in all likelihood they will need
+//! a new _unique_ event loop that addresses the needs of both languages if the coroutines are to
+//! be run on the same event loop.
+//!
+//! It's not immediately clear that this would provide worthwhile performance wins either, so in the
+//! interest of keeping things simple, this crate runs both event loops independently and handles
+//! the communication between them.
+//!
+//! ## Python's Event Loop
+//!
+//! Python is very picky about the threads used by the `asyncio` executor. In particular, it needs
+//! to have control over the main thread in order to handle signals like CTRL-C correctly. This
+//! means that Cargo's default test harness will no longer work since it doesn't provide a method of
+//! overriding the main function to add our event loop initialization and finalization.
+//!
+//! ## Rust's Event Loop
+//!
+//! Currently only the Tokio runtime is supported by this crate. Tokio makes it easy to construct
+//! and maintain a runtime that runs on background threads only and it provides a single threaded
+//! scheduler to make it easier to work around Python's GIL.
+//!
+//! > _In the future, more runtimes may be supported for Rust._
+//!
+//! ## Features
+//!
+//! Items marked with
+//! <span
+//!   class="module-item stab portability"
+//!   style="display: inline; border-radius: 3px; padding: 2px; font-size: 80%; line-height: 1.2;"
+//! ><code>testing</code></span>
+//! are only available when the `testing` Cargo feature is enabled:
+//!
+//! ```toml
+//! [dependencies.pyo3-asyncio]
+//! version = "0.1.0"
+//! features = ["testing"]
+//! ```
+
+/// Utilities for writing PyO3 Asyncio tests
+#[cfg(feature = "testing")]
+#[doc(cfg(testing))]
+#[doc(inline)]
+pub mod testing;
 
 use std::{future::Future, thread};
 
@@ -14,6 +72,25 @@ use tokio::{
     task::JoinHandle,
 };
 
+/// Test README
+#[doc(hidden)]
+pub mod doc_test {
+    macro_rules! doc_comment {
+        ($x:expr, $module:item) => {
+            #[doc = $x]
+            $module
+        };
+    }
+
+    macro_rules! doctest {
+        ($x:expr, $y:ident) => {
+            doc_comment!(include_str!($x), mod $y {});
+        };
+    }
+
+    doctest!("../README.md", readme_md);
+}
+
 lazy_static! {
     static ref CURRENT_THREAD_RUNTIME: Runtime = {
         Builder::new_current_thread()
@@ -29,10 +106,49 @@ static CALL_SOON: OnceCell<PyObject> = OnceCell::new();
 static CREATE_TASK: OnceCell<PyObject> = OnceCell::new();
 static CREATE_FUTURE: OnceCell<PyObject> = OnceCell::new();
 
+/// Wraps the provided function with the initialization and finalization for PyO3 Asyncio
+///
+/// This function **_MUST_** be called from the main thread.
+///
+/// # Arguments
+/// * `py` - The current PyO3 GIL guard
+/// * `f` - The function to call in between intialization and finalization
+///
+/// # Examples
+///
+/// ```no_run
+/// use pyo3::prelude::*;
+///
+/// fn main() {
+///     Python::with_gil(|py| {
+///         pyo3_asyncio::with_runtime(py, || {
+///             println!("PyO3 Asyncio Initialized!");
+///             Ok(())
+///         })
+///         .map_err(|e| {
+///             e.print_and_set_sys_last_vars(py);  
+///         })
+///         .unwrap();
+///     })
+/// }
+/// ```
+pub fn with_runtime<F>(py: Python, f: F) -> PyResult<()>
+where
+    F: FnOnce() -> PyResult<()>,
+{
+    try_init(py)?;
+
+    (f)()?;
+
+    try_close(py)?;
+
+    Ok(())
+}
+
 /// Attempt to initialize the Python and Rust event loops
 ///
 /// Must be called at the start of your program
-pub fn try_init(py: Python) -> PyResult<()> {
+fn try_init(py: Python) -> PyResult<()> {
     let asyncio = py.import("asyncio")?;
     let event_loop = asyncio.call_method0("get_event_loop")?;
     let executor = py
@@ -59,13 +175,12 @@ pub fn try_init(py: Python) -> PyResult<()> {
     Ok(())
 }
 
+/// Get a reference to the Python Event Loop from Rust
 pub fn get_event_loop() -> PyObject {
     EVENT_LOOP.get().unwrap().clone()
 }
 
 /// Run the event loop forever
-///
-/// This function must be called on the main thread
 pub fn run_forever(py: Python) -> PyResult<()> {
     if let Err(e) = EVENT_LOOP.get().unwrap().call_method0(py, "run_forever") {
         if e.is_instance::<PyKeyboardInterrupt>(py) {
@@ -79,8 +194,6 @@ pub fn run_forever(py: Python) -> PyResult<()> {
 }
 
 /// Run the event loop until the given Future completes
-///
-/// This function must be called on the main thread
 pub fn run_until_complete<F>(py: Python, fut: F) -> PyResult<()>
 where
     F: Future<Output = PyResult<()>> + Send + 'static,
@@ -98,7 +211,8 @@ where
     Ok(())
 }
 
-pub fn close(py: Python) -> PyResult<()> {
+/// Shutdown the event loops and perform any necessary cleanup
+fn try_close(py: Python) -> PyResult<()> {
     // Shutdown the executor and wait until all threads are cleaned up
     EXECUTOR.get().unwrap().call_method0(py, "shutdown")?;