support python 3.12

Author: baxtree

docker/Dockerfile-Dev

@@ -3,6 +3,7 @@ FROM ubuntu:20.04
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=Europe/London
+ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_SUBALIGNER=0.0.0-dev
 
 RUN mkdir -p /subaligner && chmod 755 /subaligner
 COPY subaligner /subaligner/subaligner

pyproject.toml

@@ -1,5 +1,10 @@
 [build-system]
-requires = ["setuptools<65.0.0", "setuptools-scm", "wheel"]
+requires = [
+    "setuptools<65.0.0; python_version < '3.12'",
+    "setuptools>=65.0.0; python_version >= '3.12'",
+    "setuptools-scm",
+    "wheel",
+]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -11,13 +16,14 @@ authors = [
 description = "Automatically synchronize and translate subtitles, or create new ones by transcribing, using pre-trained DNNs, Forced Alignments and Transformers."
 readme = { file = "README.md", content-type = "text/markdown" }
 urls = { "Homepage" = "https://github.com/baxtree/subaligner", "Documentation" = "https://subaligner.readthedocs.io/en/latest/", "Source" = "https://github.com/baxtree/subaligner" }
-requires-python = ">=3.8,<3.12"
+requires-python = ">=3.8,<3.13"
 classifiers = [
     "License :: OSI Approved :: MIT License",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.8",
     "Intended Audience :: Developers",
     "Topic :: Utilities"
 ]
@@ -71,9 +77,13 @@ dependencies = [
     "PyYAML>=4.2b1",
     "rsa==4.7",
     "scipy<1.12.0",
-    "scikit-learn<1.2.0",
-    "six~=1.15.0",
-    "tensorflow>=1.15.5,<2.16.0",
+    "scikit-learn<1.2.0; python_version < '3.12'",
+    "scikit-learn>=1.2.0; python_version >= '3.12'",
+    "six~=1.15.0; python_version < '3.12'",
+    "six~=1.17.0; python_version >= '3.12'",
+    "tf-keras~=2.19.0; python_version >= '3.12'",
+    "tensorflow>=1.15.5,<2.16.0; python_version < '3.12'",
+    "tensorflow~=2.19.0; python_version >= '3.12'",
     "termcolor==1.1.0",
     "toml==0.10.0",
     "toolz==0.9.0",
@@ -85,7 +95,7 @@ dependencies = [
 
 [project.optional-dependencies]
 harmony = [
-    "aeneas~=1.7.3.0",
+    "aeneas~=1.7.3.0; python_version < '3.12'",
     "dtw-python~=1.5.3",
     "sentencepiece~=0.1.95",
     "torch<2.3.0",
@@ -94,7 +104,7 @@ harmony = [
     "openai-whisper==20250625"
 ]
 dev = [
-    "aeneas~=1.7.3.0",
+    "aeneas~=1.7.3.0; python_version < '3.12'",
     "dtw-python~=1.5.3",
     "sentencepiece~=0.1.95",
     "torch<2.3.0",
@@ -105,7 +115,7 @@ dev = [
     "coverage==5.5",
     "tox~=3.23.0",
     "pycodestyle==2.12.1",
-    "twine<4.0.0",
+    "twine<3.8.0",
     "snakeviz==2.1.0",
     "line-profiler~=4.1.2",
     "scikit-build==0.11.1",
@@ -114,22 +124,24 @@ dev = [
     "mypy==1.3.0",
     "types-requests==2.27.9",
     "types-setuptools==64.0.1",
-    "typing-extensions==4.5.0",
+    "typing-extensions==4.8.0",
     "parameterized==0.8.1",
     "pylint~=2.17.2",
     "pygments<3.0.0",
     "darglint~=1.8.1"
 ]
 docs = [
     "sphinx==6.2.1",
-    "sphinx-rtd-theme==2.0.0"
+    "sphinx-rtd-theme==2.0.0",
+    "docutils<0.21",
 ]
 stretch = [
-    "aeneas~=1.7.3.0",
+    "aeneas~=1.7.3.0; python_version < '3.12'",
     "dtw-python~=1.5.3"
 ]
 llm = [
-    "sentencepiece~=0.1.95",
+    "sentencepiece~=0.1.95; python_version < '3.12'",
+    "sentencepiece~=0.2.0; python_version >= '3.12'",
     "torch<2.3.0",
     "torchaudio<2.3.0",
     "transformers<4.37.0",
@@ -171,4 +183,4 @@ exclude = "\\.git|\\.tox"
 require-return-section-when-returning-nothing = true
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.12"
+python = ">=3.8,<3.13"

subaligner/__init__.py

@@ -1,7 +1,9 @@
 import os
 import warnings
 import logging
+import sys
 import multiprocessing as mp
+import tensorflow as tf
 from ._version import __version__
 
 __all__ = ["__version__"]
@@ -16,4 +18,9 @@
 os.environ["TF_CPP_MIN_VLOG_LEVEL"] = "0"
 logging.getLogger("tensorflow").disabled = True
 
-os.environ["TF_USE_LEGACY_KERAS"] = "1"
+if tf.__version__ >= "2.16.0":
+    os.environ["TF_USE_LEGACY_KERAS"] = "1"
+    import tensorflow._api.v2.compat.v2.__internal__ as tf_internal
+    if hasattr(tf_internal, 'register_call_context_function'):
+        tf_internal.register_load_context_function = tf_internal.register_call_context_function
+    sys.modules["keras"] = tf.keras

subaligner/network.py

@@ -4,7 +4,7 @@
 import psutil
 import numpy as np
 import tensorflow as tf
-import keras.optimizers as tf_optimizers
+import tensorflow.keras.optimizers as tf_optimizers
 
 from typing import Tuple, Optional, List, Generator
 from keras.layers import (

tests/integration/feature/subaligner.feature

@@ -90,7 +90,8 @@ Feature: Subaligner CLI
 
     @with-mode @script
     Scenario Outline: Test alignments with plain texts as input
-        Given I have a video file <video-in>
+        Given I skip the scenario if Python version is above 3.11
+        And I have a video file <video-in>
         And I have a subtitle file "test_plain.txt"
         When I run the alignment with subaligner on them with script stage and output <subtitle-out>
         Then a new subtitle file <subtitle-out> is generated
@@ -181,7 +182,8 @@ Feature: Subaligner CLI
 
     @stretch
     Scenario Outline: Test dual-stage alignment with stretch on
-        Given I have a video file "test.mp4"
+        Given I skip the scenario if Python version is above 3.11
+        And I have a video file "test.mp4"
         And I have a subtitle file "test.srt"
         When I run the alignment with <aligner> on them with <mode> stage and with stretch on
         Then a new subtitle file "test_aligned.srt" is generated
@@ -244,11 +246,21 @@ Feature: Subaligner CLI
         |  aligner          |  mode     |  subtitle-in      |   language-pair   |   subtitle-out            |
         |  subaligner       |  single   |  "test.srt"       |   eng,zho         |   "test_aligned.srt"      |
         |  subaligner       |  dual     |  "test.srt"       |   eng,spa         |   "test_aligned.srt"      |
-        |  subaligner       |  script   |  "test_plain.txt" |   eng,ita         |   "test_aligned.srt"      |
-        |  subaligner       |  script   |  "test_plain.txt" |   eng,por         |   "test_aligned.srt"      |
         |  subaligner_1pass |  <NULL>   |  "test.srt"       |   eng,fra         |   "test_aligned.srt"      |
         |  subaligner_2pass |  <NULL>   |  "test.srt"       |   eng,deu         |   "test_aligned.srt"      |
 
+    @translation @script
+    Scenario Outline: Test translation on aligned scripts
+        Given I skip the scenario if Python version is above 3.11
+        And I have a video file "test.mp4"
+        And I have a subtitle file <subtitle-in>
+        When I run the alignment with <aligner> on them with <mode> stage and <language-pair> for translation
+        Then a new subtitle file <subtitle-out> is generated
+    Examples:
+        |  aligner          |  mode     |  subtitle-in      |   language-pair   |   subtitle-out            |
+        |  subaligner       |  script   |  "test_plain.txt" |   eng,ita         |   "test_aligned.srt"      |
+        |  subaligner       |  script   |  "test_plain.txt" |   eng,por         |   "test_aligned.srt"      |
+
     @transcription @custom-prompt
     Scenario Outline: Test transcription on audiovisual input and subtitle generation
         Given I have a video file <video-in>

tests/integration/radish/__init__.py

@@ -0,0 +1,2 @@
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)

tests/integration/radish/step.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import subprocess
+import sys
 import os
 import tempfile
 import shutil
@@ -105,6 +106,10 @@ def run_subaligner_manual_shift(step, offset_seconds):
 
 @when("I run the alignment with {aligner:S} on them with {mode:S} stage and {language_pair:S} for translation")
 def run_subaligner_with_translation(step, aligner, mode, language_pair):
+    if hasattr(step.context, "skip_scenario") and step.context.skip_scenario and mode == "script":
+        step.skip()
+        return
+
     if mode == "<NULL>":
         process = subprocess.Popen([
             os.path.join(PWD, "..", "..", "..", "bin", aligner),
@@ -184,6 +189,10 @@ def run_subaligner_with_transcription(step):
 
 @when('I run the alignment with {aligner:S} on them with {mode:S} stage and output "{file_name:S}"')
 def run_subaligner_with_output(step, aligner, mode, file_name):
+    if hasattr(step.context, "skip_scenario") and step.context.skip_scenario and mode == "script":
+        step.skip()
+        return
+
     if mode == "<NULL>":
         process = subprocess.Popen([
             os.path.join(PWD, "..", "..", "..", "bin", aligner),
@@ -235,8 +244,19 @@ def run_subaligner_with_exit_segfail(step, aligner, mode):
     step.context.exit_code = process.wait(timeout=WAIT_TIMEOUT_IN_SECONDS)
 
 
+@given("I skip the scenario if Python version is above {version:S}")
+def run_skip_scenario_if_python_version_above(step, version):
+    if sys.version_info > tuple([int(v) for v in version.split(".")]):
+        step.context.skip_scenario = True
+        step.skip()
+
+
 @when("I run the alignment with {aligner:S} on them with {mode:S} stage and with stretch on")
 def run_subaligner_with_stretch(step, aligner, mode):
+    if hasattr(step.context, "skip_scenario") and step.context.skip_scenario:
+        step.skip()
+        return
+
     if mode == "<NULL>":
         process = subprocess.Popen([
             os.path.join(PWD, "..", "..", "..", "bin", aligner),
@@ -279,6 +299,10 @@ def run_subaligner_with_custom_model(step, aligner, mode):
 
 @then('a new subtitle file "{file_name:S}" is generated')
 def expect_result(step, file_name):
+    if hasattr(step.context, "skip_scenario") and step.context.skip_scenario:
+        step.skip()
+        return
+
     output_file_path = os.path.join(PWD, "..", "..", "subaligner", "resource", file_name.replace("[]", " "))
     assert step.context.exit_code == 0
     assert os.path.isfile(output_file_path) is True

tests/subaligner/__init__.py

@@ -0,0 +1,5 @@
+try:
+    import aeneas
+    aeneas_is_available = True
+except ImportError:
+    aeneas_is_available = False

tests/subaligner/test_network.py

@@ -7,7 +7,6 @@
 
 from mock import patch
 from parameterized import parameterized
-from keras.models import Model
 from subaligner.hyperparameters import Hyperparameters
 from subaligner.exception import TerminalException
 from subaligner.network import Network as Undertest

tests/subaligner/test_predictor.py

@@ -8,6 +8,7 @@
 from subaligner.logger import Logger
 from subaligner.media_helper import MediaHelper
 from subaligner.predictor import Predictor as Undertest
+from tests.subaligner import aeneas_is_available
 
 
 class PredictorTests(unittest.TestCase):
@@ -249,6 +250,7 @@ def test_predict_dual_pass_with_specified_language(self):
         self.assertGreater(len(voice_probabilities), 0)
         self.assertEqual(24.0, frame_rate)
 
+    @unittest.skipIf(not aeneas_is_available, "The aeneas package is not available")
     def test_predict_plain_text(self):
         subs, audio_file_path, voice_probabilities, frame_rate = Undertest(n_mfcc=20, step_sample=0.02).predict_plain_text(
             self.video_file_path, self.plain_text_file_path