**Description**

Upgrade Apache Airflow to 3.0.4

This commit:
1. Update Apache Airflow from 3.0.1 to 3.0.4 in test_requirements.txt
2. Update Apache Airflow providers to 8.30.0
3. Add Flask-Limiter 2.9.2 to resolve dependency conflicts
4. Modify tox.ini to use development mode installation for sphinx

These changes allow the documentation to build successfully while using the latest Apache Airflow version.

**Testing Done**

Test the command ```tox -e sphinx --parallel all``` and it succeeded.

Author: papriwal

doc/requirements.txt

@@ -1,7 +1,7 @@
 sphinx==7.2.6
 sphinx-rtd-theme==3.0.0
 docutils>=0.18.1,<0.21
-packaging>=23.0,<25
+packaging>=24.0,<25
 jinja2==3.1.6
 schema==0.7.5
 accelerate>=0.24.1,<=0.27.0

pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
   "psutil",
   "PyYAML>=6.0.1",
   "requests",
-  "sagemaker-core>=1.0.17,<2.0.0",
+  "sagemaker-core>=1.0.42,<2.0.0",
   "schema",
   "smdebug_rulesconfig==1.0.1",
   "tblib>=1.7.0,<4",

requirements/constraints.txt

@@ -0,0 +1,22 @@
+# Constraints file to resolve dependency conflicts
+# This forces specific versions to be used regardless of what other packages require
+
+# Core conflicts
+importlib-metadata<7.0
+rich>=13.6.0,<14.0.0
+Flask-Limiter==2.9.2
+
+# Additional constraints to ensure compatibility
+sagemaker-core==1.0.42
+apache-airflow==3.0.4
+apache-airflow-providers-amazon==8.30.0
+
+# Sphinx dependencies
+sphinx==7.2.6
+sphinx-rtd-theme==3.0.0
+docutils>=0.18.1,<0.21
+packaging>=23.0,<25
+jinja2==3.1.6
+schema==0.7.5
+accelerate>=0.24.1,<=0.27.0
+graphene<4.0

requirements/extras/test_requirements.txt

@@ -14,9 +14,9 @@ awslogs==0.14.0
 black==24.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==3.0.2
-apache-airflow-providers-amazon==7.2.1
-Flask-Limiter==3.12
+apache-airflow==3.0.4
+apache-airflow-providers-amazon==8.30.0
+Flask-Limiter==2.9.2
 attrs>=24,<26
 fabric==3.2.2
 requests==2.32.2

src/sagemaker/serve/utils/conda_in_process.yml

@@ -15,7 +15,7 @@ dependencies:
     - numpy==1.26.4
     - protobuf>=3.12,<5.0
     - smdebug_rulesconfig==1.0.1
-    - importlib-metadata>=1.4.0,<9.0
+    - importlib-metadata>=1.4.0,<7.0
     - packaging>=23.0,<25
     - pandas
     - pathos
@@ -85,7 +85,8 @@ dependencies:
             - pyyaml>=6.0.1
             - regex>=2023.3.23
             - requests>=2.28.2
-            - rich>=13.3.4
+            - rich>=13.3.4,<14.0.0
+            - safetensors>=0.4.0
             - rsa>=4.7.2
             - s3transfer>=0.6.0
             - sagemaker>=2.148.0

src/sagemaker/serve/utils/in_process_requirements.txt

@@ -53,7 +53,7 @@ pytz-deprecation-shim>=0.1.0.post0
 pyyaml>=6.0.1
 regex>=2023.3.23
 requests>=2.28.2
-rich>=13.3.4
+rich>=13.3.4,<14.0.0
 rsa>=4.7.2
 s3transfer>=0.6.0
 sagemaker>=2.148.0

tox.ini

@@ -82,14 +82,16 @@ passenv =
 # Can be used to specify which tests to run, e.g.: tox -- -s
 commands =
     python -c "import os; os.system('install-custom-pkgs --install-boto-wheels')"
-    pip install 'apache-airflow==3.0.2' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.2/constraints-3.9.txt"
+    pip install 'apache-airflow==3.0.4' --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.0.4/constraints-3.9.txt"
     pip install 'torch==2.3.1+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'torchvision==0.18.1+cpu' -f 'https://download.pytorch.org/whl/torch_stable.html'
     pip install 'dill>=0.3.9'
     pip install 'altair>=5.3' # needed for amtviz
 
     pytest {posargs}
-deps = .[test]
+deps =
+    -c {toxinidir}/requirements/constraints.txt
+    .[test]
 depends =
     {py39,py310,py311,py312}: clean
 
@@ -138,12 +140,22 @@ commands =
 
 [testenv:sphinx]
 pip_version = pip==24.3
+skip_install = true
 changedir = doc
 # pip install requirements.txt is separate as RTD does it in separate steps
 # having the requirements.txt installed in deps above results in Double Requirement exception
 # https://github.com/pypa/pip/issues/988
+deps =
+    sphinx==7.2.6
+    sphinx-rtd-theme==3.0.0
+    docutils>=0.18.1,<0.21
+    packaging>=23.0,<25
+    jinja2==3.1.6
+    schema==0.7.5
+    accelerate>=0.24.1,<=0.27.0
+    graphene<4.0
 commands =
-    pip install --exists-action=w -r requirements.txt
+    pip install -e {toxinidir}
     sphinx-build -T -b html -d _build/doctrees-readthedocs -D language=en . _build/html
 
 [testenv:doc8]
@@ -198,6 +210,8 @@ commands =
 
 [testenv:collect-tests]
 # this needs to succeed for tests to display in some IDEs
-deps = .[test]
+skip_install = true
+deps =
+    pytest==6.2.5
 commands =
     pytest --collect-only