diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000..560b62278 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,32 @@ +# Note: any test whose name starts with 'e2e' is considered an end-to-end test +# and will be included or excluded based on the profile used. + +nextest-version = "0.9.115" + +[profile.default] +default-filter = "not test(test_e2e)" +# Skip end-to-end tests by default + +[profile.e2e] +default-filter = "all()" + +[profile.ci] +default-filter = "all()" +slow-timeout = "20s" +fail-fast = false + +[profile.ci.junit] +path = "junit.xml" + +[profile.ci-skip-e2e] +inherits = "ci" +default-filter = "not test(test_e2e)" + +[[profile.default.overrides]] +filter = "test(test_e2e)" +test-group = "e2e" +retries = { backoff = "fixed", count = 4, delay = "1s" } +slow-timeout = "60s" + +[test-groups.e2e] +max-threads = 1 # Run end-to-end tests serially \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 074453674..d7ddbf995 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,37 +20,35 @@ on: pull_request: branches: ["main"] -env: - CARGO_TERM_COLOR: always - RUST_LOG: debug - jobs: - build: + test: + name: Test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 strategy: fail-fast: false matrix: include: - # Platform supports is limited for tests (no aarch64) - - os: windows-latest - target: x86_64-pc-windows-msvc - - os: macos-latest - target: x86_64-apple-darwin - e2e-testing: true - - os: ubuntu-latest - target: x86_64-unknown-linux-gnu - e2e-testing: true - - name: Test (${{ matrix.target }}) - runs-on: ${{ matrix.os }} + - { os: macos-latest, test-profile: ci } + - { os: ubuntu-latest, test-profile: ci } + - { os: windows-latest, test-profile: ci-skip-e2e } + permissions: + contents: read + actions: read + checks: write + # Required for test reporting steps: - - uses: actions/checkout@v3 - with: - submodules: true + - name: Checkout + uses: actions/checkout@v3 + with: { submodules: true } - - name: Install Rust toolchain - run: | - rustup update --no-self-update nightly - rustup target add ${{ matrix.target }} --toolchain nightly + - name: Setup Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: { rustflags: "" } # Don't fail on warnings + + - name: Install Nextest + uses: taiki-e/install-action@v2 + with: { tool: nextest@0.9.115 } - name: Install Protoc uses: arduino/setup-protoc@v2 @@ -58,26 +56,6 @@ jobs: version: "25.2" repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache Cargo registry - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-registry- - - - name: Cache Cargo build - uses: actions/cache@v3 - with: - path: target/ - key: ${{ runner.os }}-cargo-build-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo-build-${{ matrix.target }}- - ${{ runner.os }}-cargo-build- - - name: Install linux dependencies if: ${{ matrix.os == 'ubuntu-latest' }} run: | @@ -103,7 +81,7 @@ jobs: libglib2.0-dev - name: Install LiveKit server - if: ${{ matrix.e2e-testing }} + if: ${{ matrix.test-profile == 'ci' }} run: | if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then curl -sSL https://get.livekit.io | bash @@ -112,17 +90,18 @@ jobs: fi - name: Run LiveKit server - if: ${{ matrix.e2e-testing }} + if: ${{ matrix.test-profile == 'ci' }} run: livekit-server --dev & - - name: Test (no E2E) - if: ${{ !matrix.e2e-testing }} + - name: Test (${{ matrix.test-profile }}) env: RUST_LOG: info - run: cargo +nightly test --release --verbose --target ${{ matrix.target }} -- --nocapture + run: cargo nextest run -P ${{ matrix.test-profile }} --no-capture - - name: Test (with E2E) - if: ${{ matrix.e2e-testing }} - env: - RUST_LOG: info - run: cargo +nightly test --release --verbose --target ${{ matrix.target }} --features __lk-e2e-test -- --nocapture + - name: Test Report + uses: dorny/test-reporter@v2 + if: ${{ !cancelled() }} + with: + name: Test Results + path: target/nextest/${{ matrix.test-profile }}/junit.xml + reporter: java-junit \ No newline at end of file diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index 71124f738..8c9ed1021 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -23,7 +23,6 @@ rustls-tls-native-roots = ["livekit-api/rustls-tls-native-roots"] rustls-tls-webpki-roots = ["livekit-api/rustls-tls-webpki-roots"] __rustls-tls = ["livekit-api/__rustls-tls"] __lk-internal = [] # internal features (used by livekit-ffi) -__lk-e2e-test = [] # end-to-end testing with a LiveKit server [dependencies] livekit-runtime = { workspace = true } diff --git a/livekit/tests/audio_test.rs b/livekit/tests/audio_test.rs index 869f3ec64..89778bd3e 100644 --- a/livekit/tests/audio_test.rs +++ b/livekit/tests/audio_test.rs @@ -12,19 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] -use { - anyhow::{anyhow, Ok, Result}, - common::{ - audio::{ChannelIterExt, FreqAnalyzer, SineParameters, SineTrack}, - test_rooms, - }, - futures_util::StreamExt, - libwebrtc::audio_stream::native::NativeAudioStream, - livekit::prelude::*, - std::{sync::Arc, time::Duration}, - tokio::time::timeout, +use anyhow::{anyhow, Ok, Result}; +use common::{ + audio::{ChannelIterExt, FreqAnalyzer, SineParameters, SineTrack}, + test_rooms, }; +use futures_util::StreamExt; +use libwebrtc::audio_stream::native::NativeAudioStream; +use livekit::prelude::*; +use std::{sync::Arc, time::Duration}; +use tokio::time::timeout; mod common; @@ -35,9 +32,8 @@ struct TestParams { sub_channels: u32, } -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -async fn test_audio() -> Result<()> { +async fn test_e2e_audio() -> Result<()> { let test_params = [ TestParams { pub_rate_hz: 48_000, pub_channels: 1, sub_rate_hz: 48_000, sub_channels: 1 }, TestParams { pub_rate_hz: 48_000, pub_channels: 2, sub_rate_hz: 48_000, sub_channels: 2 }, @@ -56,7 +52,6 @@ async fn test_audio() -> Result<()> { /// Verifies that audio can be published and received correctly /// between two participants by detecting the frequency of the sine wave on the subscriber end. /// -#[cfg(feature = "__lk-e2e-test")] async fn test_audio_with(params: TestParams) -> Result<()> { let mut rooms = test_rooms(2).await?; let (pub_room, _) = rooms.pop().unwrap(); diff --git a/livekit/tests/common/mod.rs b/livekit/tests/common/mod.rs index e6bb827c9..f40a5fecf 100644 --- a/livekit/tests/common/mod.rs +++ b/livekit/tests/common/mod.rs @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] /// Utilities for end-to-end testing with a LiveKit server. mod e2e; -#[cfg(feature = "__lk-e2e-test")] pub use e2e::*; diff --git a/livekit/tests/data_channel_encryption.rs b/livekit/tests/data_channel_encryption.rs index 2e1515423..8d87b635a 100644 --- a/livekit/tests/data_channel_encryption.rs +++ b/livekit/tests/data_channel_encryption.rs @@ -12,26 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] -use { - crate::common::test_rooms_with_options, - anyhow::{Ok, Result}, - livekit::{ - e2ee::{ - key_provider::{KeyProvider, KeyProviderOptions}, - EncryptionType, - }, - DataPacket, E2eeOptions, RoomEvent, RoomOptions, +use anyhow::{Ok, Result}; +use common::test_rooms_with_options; +use livekit::{ + e2ee::{ + key_provider::{KeyProvider, KeyProviderOptions}, + EncryptionType, }, - std::time::Duration, - tokio::{time::timeout, try_join}, + DataPacket, E2eeOptions, RoomEvent, RoomOptions, }; +use std::time::Duration; +use tokio::{time::timeout, try_join}; mod common; -#[cfg(feature = "__lk-e2e-test")] #[tokio::test] -async fn test_data_channel_encryption() -> Result<()> { +async fn test_e2e_data_channel_encryption() -> Result<()> { const ITERATIONS: usize = 128; const PAYLOAD_SIZE: usize = 4096; diff --git a/livekit/tests/data_channel_test.rs b/livekit/tests/data_channel_test.rs index af38a551d..3cdbd8898 100644 --- a/livekit/tests/data_channel_test.rs +++ b/livekit/tests/data_channel_test.rs @@ -12,23 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] -use { - crate::common::test_rooms, - anyhow::{Ok, Result}, - livekit::{DataPacket, RoomEvent, SimulateScenario}, - std::{sync::Arc, time::Duration}, - tokio::{ - time::{self, timeout}, - try_join, - }, +use anyhow::{Ok, Result}; +use common::test_rooms; +use livekit::{DataPacket, RoomEvent, SimulateScenario}; +use std::{sync::Arc, time::Duration}; +use tokio::{ + time::{self, timeout}, + try_join, }; mod common; -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -async fn test_reliable_retry() -> Result<()> { +async fn test_e2e_reliable_retry() -> Result<()> { use anyhow::Context; const ITERATIONS: usize = 128; diff --git a/livekit/tests/data_stream_test.rs b/livekit/tests/data_stream_test.rs index 911a37548..03c083985 100644 --- a/livekit/tests/data_stream_test.rs +++ b/livekit/tests/data_stream_test.rs @@ -12,21 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] -use { - crate::common::test_rooms, - anyhow::{anyhow, Ok, Result}, - chrono::{TimeDelta, Utc}, - livekit::{RoomEvent, StreamByteOptions, StreamReader, StreamTextOptions}, - std::time::Duration, - tokio::{time::timeout, try_join}, -}; +use anyhow::{anyhow, Ok, Result}; +use chrono::{TimeDelta, Utc}; +use common::test_rooms; +use livekit::{RoomEvent, StreamByteOptions, StreamReader, StreamTextOptions}; +use std::time::Duration; +use tokio::{time::timeout, try_join}; mod common; -#[cfg(feature = "__lk-e2e-test")] #[tokio::test] -async fn test_send_bytes() -> Result<()> { +async fn test_e2e_send_bytes() -> Result<()> { let mut rooms = test_rooms(2).await?; let (sending_room, _) = rooms.pop().unwrap(); let (_, mut receiving_event_rx) = rooms.pop().unwrap(); @@ -70,9 +66,8 @@ async fn test_send_bytes() -> Result<()> { Ok(()) } -#[cfg(feature = "__lk-e2e-test")] #[tokio::test] -async fn test_send_text() -> Result<()> { +async fn test_e2e_send_text() -> Result<()> { let mut rooms = test_rooms(2).await?; let (sending_room, _) = rooms.pop().unwrap(); let (_, mut receiving_event_rx) = rooms.pop().unwrap(); diff --git a/livekit/tests/room_test.rs b/livekit/tests/room_test.rs index ea944046d..03daeefbf 100644 --- a/livekit/tests/room_test.rs +++ b/livekit/tests/room_test.rs @@ -12,21 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] -use { - anyhow::{Ok, Result}, - chrono::{TimeDelta, TimeZone, Utc}, - common::test_rooms, - livekit::{ConnectionState, ParticipantKind, RoomEvent}, - std::time::Duration, - tokio::time::{self, timeout}, -}; +use anyhow::{Ok, Result}; +use chrono::{TimeDelta, TimeZone, Utc}; +use common::test_rooms; +use livekit::{ConnectionState, ParticipantKind, RoomEvent}; +use std::time::Duration; +use tokio::time::{self, timeout}; mod common; -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -async fn test_connect() -> Result<()> { +async fn test_e2e_connect() -> Result<()> { let (room, _) = test_rooms(1).await?.pop().unwrap(); assert_eq!(room.connection_state(), ConnectionState::Connected); @@ -45,9 +41,8 @@ async fn test_connect() -> Result<()> { Ok(()) } -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -async fn test_connect_multiple() -> Result<()> { +async fn test_e2e_connect_multiple() -> Result<()> { let mut rooms = test_rooms(2).await?; let (second, _) = rooms.pop().unwrap(); @@ -61,9 +56,8 @@ async fn test_connect_multiple() -> Result<()> { Ok(()) } -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -async fn test_participant_disconnect() -> Result<()> { +async fn test_e2e_participant_disconnect() -> Result<()> { let mut rooms = test_rooms(2).await?; let (second, _) = rooms.pop().unwrap(); let second_sid = second.local_participant().sid(); diff --git a/livekit/tests/rpc_test.rs b/livekit/tests/rpc_test.rs index edee6063d..59b8004cb 100644 --- a/livekit/tests/rpc_test.rs +++ b/livekit/tests/rpc_test.rs @@ -12,19 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(feature = "__lk-e2e-test")] -use { - anyhow::{Context, Result}, - common::test_rooms, - livekit::prelude::PerformRpcData, - std::time::Duration, -}; +use anyhow::{Context, Result}; +use common::test_rooms; +use livekit::prelude::PerformRpcData; +use std::time::Duration; mod common; -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -pub async fn test_rpc_invocation() -> Result<()> { +pub async fn test_e2e_rpc_invocation() -> Result<()> { let mut rooms = test_rooms(2).await?; let (caller_room, _) = rooms.pop().unwrap(); let (callee_room, _) = rooms.pop().unwrap(); @@ -54,9 +50,8 @@ pub async fn test_rpc_invocation() -> Result<()> { Ok(()) } -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -pub async fn test_rpc_unregistered() -> Result<()> { +pub async fn test_e2e_rpc_unregistered() -> Result<()> { let mut rooms = test_rooms(2).await?; let (caller_room, _) = rooms.pop().unwrap(); let (callee_room, _) = rooms.pop().unwrap(); @@ -77,9 +72,8 @@ pub async fn test_rpc_unregistered() -> Result<()> { Ok(()) } -#[cfg(feature = "__lk-e2e-test")] #[test_log::test(tokio::test)] -pub async fn test_rpc_unknown_destination() -> Result<()> { +pub async fn test_e2e_rpc_unknown_destination() -> Result<()> { let mut rooms = test_rooms(1).await?; let (caller_room, _) = rooms.pop().unwrap();