Skip to content

Remove remaining tests from collection #1142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mollyxu merged 15 commits into from
Jan 12, 2026
Merged

Remove remaining tests from collection #1142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
d904649
skip remaining tests
Dec 17, 2025
bf7613a
fix tests
Dec 18, 2025
e3b484c
fix tests
Dec 18, 2025
5568e4b
fix tests
Dec 22, 2025
11eaa4d
fix tests
Dec 22, 2025
fc5a1bf
fix tests
Dec 23, 2025
8388c54
testing
Dec 23, 2025
6e816d8
fix tests
Dec 23, 2025
7e6f4f1
modify parametrize logic
Jan 5, 2026
2decbd3
fix formatting
Jan 5, 2026
3fa1345
address feedback
Jan 9, 2026
96f498b
sync
Jan 9, 2026
5ec71ba
Merge branch 'main' into skip-tests-internally
mollyxu Jan 9, 2026
8af2c8d
fix bug
Jan 9, 2026
076c987
fix bug
Jan 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,15 @@ def pytest_collection_modifyitems(items):
skipif_marker.args[0] for skipif_marker in item.iter_markers("skipif")
)

# If we need to conditionally skip tests based on a dependency, we should follow
# the decorator pattern used by needs_cuda and needs_ffmpeg_cli:
# 1. Define a custom marker in pytest_configure() above
# 2. Create a decorator function in utils.py (e.g., @needs_my_dependency)
# 3. Handle the marker here in pytest_collection_modifyitems()
# This keeps our skip logic centralized

if in_fbcode():
# fbcode doesn't like skipping tests, so instead we just don't collect the test
# fbcode doesn't like skipping tests, so instead we just don't collect the test
# so that they don't even "exist", hence the continue statements.
if needs_ffmpeg_cli or has_skip_marker or skipif_condition_is_true:
continue
Expand Down
79 changes: 38 additions & 41 deletions test/test_decoders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1191,24 +1191,22 @@ def get_some_frames(decoder):
assert_frames_equal(ref_frame3, frames[1].data)
assert_frames_equal(ref_frame5, frames[2].data)

# The test video we have is from
# https://huggingface.co/datasets/raushan-testing-hf/videos-test/blob/main/sample_video_2.avi
# We can't check it into the repo due to potential licensing issues, so
# we have to unconditionally skip this test.
# TODO: encode a video with no pts values to unskip this test. Couldn't
# find a way to do that with FFmpeg's CLI, but this should be doable
# once we have our own video encoder.
@pytest.mark.parametrize("seek_mode", ("exact", "approximate"))
@pytest.mark.skip(reason="TODO: Need video with no pts values.")
def test_pts_to_dts_fallback(self, seek_mode):
# Non-regression test for
# https://github.com/pytorch/torchcodec/issues/677 and
# https://github.com/pytorch/torchcodec/issues/676.
# More accurately, this is a non-regression test for videos which do
# *not* specify pts values (all pts values are N/A and set to
# INT64_MIN), but specify *dts* value - which we fallback to.
#
# The test video we have is from
# https://huggingface.co/datasets/raushan-testing-hf/videos-test/blob/main/sample_video_2.avi
# We can't check it into the repo due to potential licensing issues, so
# we have to unconditionally skip this test.#
# TODO: encode a video with no pts values to unskip this test. Couldn't
# find a way to do that with FFmpeg's CLI, but this should be doable
# once we have our own video encoder.
pytest.skip(reason="TODO: Need video with no pts values.")

path = "/home/nicolashug/Downloads/sample_video_2.avi"
decoder = VideoDecoder(path, seek_mode=seek_mode)
metadata = decoder.metadata
Expand Down Expand Up @@ -1295,16 +1293,6 @@ def test_10bit_videos(self, device, asset):
# This just validates that we can decode 10-bit videos.
# TODO validate against the ref that the decoded frames are correct

if device == "cuda:beta" and asset is H264_10BITS:
Copy link
Contributor Author

@mollyxu mollyxu Jan 5, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no longer applicable because the BETA interface fallsback to the CPU now

# This fails on the BETA interface with:
#
# RuntimeError: Codec configuration not supported on this GPU.
# Codec: 4, chroma format: 1, bit depth: 10
#
# It works on the ffmpeg interface because FFmpeg fallsback to the
# CPU, while the BETA interface doesn't.
pytest.skip("Asset not supported by NVDEC")

decoder, _ = make_video_decoder(asset.path, device=device)
decoder.get_frame_at(10)

Expand Down Expand Up @@ -1456,7 +1444,12 @@ def test_get_frames_at_tensor_indices(self):
TEST_SRC_2_720P,
BT709_FULL_RANGE,
TEST_SRC_2_720P_H265,
AV1_VIDEO,
pytest.param(
AV1_VIDEO,
marks=pytest.mark.skipif(
in_fbcode(), reason="AV1 CUDA not supported internally"
),
),
TEST_SRC_2_720P_VP9,
TEST_SRC_2_720P_VP8,
TEST_SRC_2_720P_MPEG4,
Expand All @@ -1467,10 +1460,6 @@ def test_get_frames_at_tensor_indices(self):
def test_beta_cuda_interface_get_frame_at(
self, asset, contiguous_indices, seek_mode
):

if in_fbcode() and asset is AV1_VIDEO:
pytest.skip("AV1 CUDA not supported internally")

ref_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
with set_cuda_backend("beta"):
beta_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
Expand Down Expand Up @@ -1502,7 +1491,12 @@ def test_beta_cuda_interface_get_frame_at(
TEST_SRC_2_720P,
BT709_FULL_RANGE,
TEST_SRC_2_720P_H265,
AV1_VIDEO,
pytest.param(
AV1_VIDEO,
marks=pytest.mark.skipif(
in_fbcode(), reason="AV1 CUDA not supported internally"
),
),
TEST_SRC_2_720P_VP9,
TEST_SRC_2_720P_VP8,
TEST_SRC_2_720P_MPEG4,
Expand All @@ -1513,9 +1507,6 @@ def test_beta_cuda_interface_get_frame_at(
def test_beta_cuda_interface_get_frames_at(
self, asset, contiguous_indices, seek_mode
):
if in_fbcode() and asset is AV1_VIDEO:
pytest.skip("AV1 CUDA not supported internally")

ref_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
with set_cuda_backend("beta"):
beta_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
Expand Down Expand Up @@ -1548,17 +1539,19 @@ def test_beta_cuda_interface_get_frames_at(
TEST_SRC_2_720P,
BT709_FULL_RANGE,
TEST_SRC_2_720P_H265,
AV1_VIDEO,
pytest.param(
AV1_VIDEO,
marks=pytest.mark.skipif(
in_fbcode(), reason="AV1 CUDA not supported internally"
),
),
TEST_SRC_2_720P_VP9,
TEST_SRC_2_720P_VP8,
TEST_SRC_2_720P_MPEG4,
),
)
@pytest.mark.parametrize("seek_mode", ("exact", "approximate"))
def test_beta_cuda_interface_get_frame_played_at(self, asset, seek_mode):
if in_fbcode() and asset is AV1_VIDEO:
pytest.skip("AV1 CUDA not supported internally")

ref_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
with set_cuda_backend("beta"):
beta_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
Expand Down Expand Up @@ -1588,17 +1581,19 @@ def test_beta_cuda_interface_get_frame_played_at(self, asset, seek_mode):
TEST_SRC_2_720P,
BT709_FULL_RANGE,
TEST_SRC_2_720P_H265,
AV1_VIDEO,
pytest.param(
AV1_VIDEO,
marks=pytest.mark.skipif(
in_fbcode(), reason="AV1 CUDA not supported internally"
),
),
TEST_SRC_2_720P_VP9,
TEST_SRC_2_720P_VP8,
TEST_SRC_2_720P_MPEG4,
),
)
@pytest.mark.parametrize("seek_mode", ("exact", "approximate"))
def test_beta_cuda_interface_get_frames_played_at(self, asset, seek_mode):
if in_fbcode() and asset is AV1_VIDEO:
pytest.skip("AV1 CUDA not supported internally")

ref_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
with set_cuda_backend("beta"):
beta_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
Expand Down Expand Up @@ -1629,17 +1624,19 @@ def test_beta_cuda_interface_get_frames_played_at(self, asset, seek_mode):
TEST_SRC_2_720P,
BT709_FULL_RANGE,
TEST_SRC_2_720P_H265,
AV1_VIDEO,
pytest.param(
AV1_VIDEO,
marks=pytest.mark.skipif(
in_fbcode(), reason="AV1 CUDA not supported internally"
),
),
TEST_SRC_2_720P_VP9,
TEST_SRC_2_720P_VP8,
TEST_SRC_2_720P_MPEG4,
),
)
@pytest.mark.parametrize("seek_mode", ("exact", "approximate"))
def test_beta_cuda_interface_backwards(self, asset, seek_mode):
if in_fbcode() and asset is AV1_VIDEO:
pytest.skip("AV1 CUDA not supported internally")

ref_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
with set_cuda_backend("beta"):
beta_decoder = VideoDecoder(asset.path, device="cuda", seek_mode=seek_mode)
Expand Down
53 changes: 37 additions & 16 deletions test/test_encoders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,27 @@ def test_round_trip(self, method, format, tmp_path):
@pytest.mark.parametrize("bit_rate", (None, 0, 44_100, 999_999_999))
@pytest.mark.parametrize("num_channels", (None, 1, 2))
@pytest.mark.parametrize("sample_rate", (8_000, 32_000))
@pytest.mark.parametrize("format", ("mp3", "wav", "flac"))
@pytest.mark.parametrize(
"format",
[
# TODO: https://github.com/pytorch/torchcodec/issues/837
pytest.param(
"mp3",
marks=pytest.mark.skipif(
IS_WINDOWS and get_ffmpeg_major_version() <= 5,
reason="Encoding mp3 on Windows is weirdly buggy",
),
),
pytest.param(
"wav",
marks=pytest.mark.skipif(
get_ffmpeg_major_version() == 4,
reason="Swresample with FFmpeg 4 doesn't work on wav files",
),
),
"flac",
],
)
@pytest.mark.parametrize("method", ("to_file", "to_tensor", "to_file_like"))
def test_against_cli(
self,
Expand All @@ -283,12 +303,6 @@ def test_against_cli(
# Encodes samples with our encoder and with the FFmpeg CLI, and checks
# that both decoded outputs are equal

if get_ffmpeg_major_version() == 4 and format == "wav":
pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files")
if IS_WINDOWS and get_ffmpeg_major_version() <= 5 and format == "mp3":
# TODO: https://github.com/pytorch/torchcodec/issues/837
pytest.skip("Encoding mp3 on Windows is weirdly buggy")

encoded_by_ffmpeg = tmp_path / f"ffmpeg_output.{format}"
subprocess.run(
["ffmpeg", "-i", str(asset.path)]
Expand Down Expand Up @@ -845,13 +859,15 @@ def test_extra_options_errors(self, method, tmp_path, extra_options, error):
pytest.mark.skipif(
in_fbcode(), reason="NVENC not available in fbcode"
),
pytest.mark.skipif(
get_ffmpeg_major_version() == 4,
reason="CUDA + FFmpeg 4 test is flaky",
),
],
),
),
)
def test_contiguity(self, method, tmp_path, device):
if get_ffmpeg_major_version() == 4 and device == "cuda":
pytest.skip("CUDA + FFmpeg 4 test is flaky")
# Ensure that 2 sets of video frames with the same pixel values are encoded
# in the same way, regardless of their memory layout. Here we encode 2 equal
# frame tensors, one is contiguous while the other is non-contiguous.
Expand Down Expand Up @@ -1014,7 +1030,17 @@ def test_against_to_file(self, tmp_path, format, method):
"avi",
"mkv",
"flv",
pytest.param("webm", marks=pytest.mark.slow),
pytest.param(
"webm",
marks=[
pytest.mark.slow,
pytest.mark.skipif(
get_ffmpeg_major_version() == 4
or (IS_WINDOWS and get_ffmpeg_major_version() in (6, 7)),
reason="Codec for webm is not available in this FFmpeg installation.",
),
],
),
),
)
@pytest.mark.parametrize(
Expand All @@ -1031,15 +1057,10 @@ def test_against_to_file(self, tmp_path, format, method):
def test_video_encoder_against_ffmpeg_cli(
self, tmp_path, format, encode_params, method, frame_rate
):
ffmpeg_version = get_ffmpeg_major_version()
if format == "webm" and (
ffmpeg_version == 4 or (IS_WINDOWS and ffmpeg_version in (6, 7))
):
pytest.skip("Codec for webm is not available in this FFmpeg installation.")

pixel_format = encode_params["pixel_format"]
crf = encode_params["crf"]
preset = encode_params["preset"]
ffmpeg_version = get_ffmpeg_major_version()

if format in ("avi", "flv") and pixel_format == "yuv444p":
pytest.skip(f"Default codec for {format} does not support {pixel_format}")
Expand Down
12 changes: 9 additions & 3 deletions test/test_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,15 @@ def _get_container_metadata(path, seek_mode):
get_container_metadata_from_header,
functools.partial(_get_container_metadata, seek_mode="approximate"),
functools.partial(_get_container_metadata, seek_mode="exact"),
functools.partial(_get_container_metadata, seek_mode="custom_frame_mappings"),
pytest.param(
functools.partial(
_get_container_metadata, seek_mode="custom_frame_mappings"
),
marks=pytest.mark.skipif(
get_ffmpeg_major_version() in (4, 5),
reason="ffprobe isn't accurate on ffmpeg 4 and 5",
),
),
),
)
def test_get_metadata(metadata_getter):
Expand All @@ -57,8 +65,6 @@ def test_get_metadata(metadata_getter):
if isinstance(metadata_getter, functools.partial)
else None
)
if (seek_mode == "custom_frame_mappings") and get_ffmpeg_major_version() in (4, 5):
pytest.skip(reason="ffprobe isn't accurate on ffmpeg 4 and 5")
metadata = metadata_getter(NASA_VIDEO.path)

with_scan = (
Expand Down
Loading