Merge branch 'main' into remove_unused_code

Author: svekars

.ci/docker/requirements.txt

@@ -14,7 +14,7 @@ tqdm==4.66.1
 numpy==1.24.4
 matplotlib
 librosa
-torch==2.6
+torch==2.7
 torchvision
 torchdata
 networkx
@@ -28,15 +28,15 @@ tensorboard
 jinja2==3.1.3
 pytorch-lightning
 torchx
-torchrl==0.6.0
-tensordict==0.6.0
+torchrl==0.7.2
+tensordict==0.7.2
 ax-platform>=0.4.0
 nbformat>=5.9.2
 datasets
 transformers
 torchmultimodal-nightly # needs to be updated to stable as soon as it's avaialable
 onnx
-onnxscript
+onnxscript>=0.2.2
 onnxruntime
 evaluate
 accelerate>=0.20.1
@@ -67,7 +67,7 @@ iopath
 pygame==2.6.0
 pycocotools
 semilearn==0.3.2
-torchao==0.5.0
+torchao==0.10.0
 segment_anything==1.0
-torchrec==1.0.0; platform_system == "Linux"
-fbgemm-gpu==1.1.0; platform_system == "Linux"
+torchrec==1.1.0; platform_system == "Linux"
+fbgemm-gpu==1.2.0; platform_system == "Linux"

.github/workflows/MonthlyLinkCheck.yml

@@ -0,0 +1,44 @@
+#Runs once a month and checks links in the repo to ensure they are valid 
+#If action fails, it creates an issue with the failing links and an "incorrect link" label
+#If link is valid but failing, it can be added to the .lycheeignore file
+#Action can also be run manually as needed.
+
+
+name: Monthly Link Check
+on:
+  schedule:
+    - cron: '0 0 1 * *'  # Runs at midnight on the first day of every month
+  workflow_dispatch:  # Allows manual triggering of the workflow
+jobs:
+  linkChecker:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Check Links
+        id: lychee
+        uses: lycheeverse/lychee-action@v2
+        with:
+          args: --accept=200,403,429 --base . --verbose --no-progress './**/*.md' './**/*.html' './**/*.rst'
+          token: ${{ secrets.CUSTOM_TOKEN }}
+          fail: true
+
+      - name: Create Issue From File
+        if: failure() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        uses: peter-evans/create-issue-from-file@v5
+        with:
+          title: Broken links detected in docs 🔗
+          content-filepath: ./lychee/out.md
+          labels: 'incorrect link'
+          #token: ${{ secrets.CUSTOM_TOKEN }}
+          
+    
+      - name: Suggestions
+        if: failure()
+        run: |
+          echo -e "\nPlease review the links reported in the Check links step above."
+          echo -e "If a link is valid but fails due to a CAPTCHA challenge, IP blocking, login requirements, etc., consider adding such links to .lycheeignore file to bypass future checks.\n"
+          exit 1

.github/workflows/build-tutorials.yml

@@ -22,15 +22,15 @@ jobs:
           - { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
           - { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
           - { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
-          - { shard: 7, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 8, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 9, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 10, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 11, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 12, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 13, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 14, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
-          - { shard: 15, num_shards: 15, runner: "linux.4xlarge.nvidia.gpu" }
+          - { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
+          - { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
       fail-fast: false
     runs-on: ${{ matrix.runner }}
     steps:

.github/workflows/link_checkPR.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Get Changed Files
         id: changed-files
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@d6e91a2266cdb9d62096cebf1e8546899c6aa18f # v45.0.6
 
       - name: Check for Skip Label
         id: skip-label

.jenkins/build.sh

@@ -22,11 +22,10 @@ sudo apt-get install -y pandoc
 #Install PyTorch Nightly for test.
 # Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
 # Install 2.5 to merge all 2.4 PRs - uncomment to install nightly binaries (update the version as needed).
-# sudo pip uninstall -y torch torchvision torchaudio torchtext torchdata
-# sudo pip3 install torch==2.6.0 torchvision --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu124
 # sudo pip uninstall -y fbgemm-gpu torchrec
+# sudo pip uninstall -y torch torchvision torchaudio torchtext torchdata torchrl tensordict
 # sudo pip3 install fbgemm-gpu==1.1.0 torchrec==1.0.0 --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu124
-
+# pip3 install torch==2.7.0 torchvision torchaudio --no-cache-dir --index-url https://download.pytorch.org/whl/test/cu126
 # Install two language tokenizers for Translation with TorchText tutorial
 python -m spacy download en_core_web_sm
 python -m spacy download de_core_news_sm

.jenkins/post_process_notebooks.py

@@ -12,7 +12,7 @@
 
 
 # Pattern to search ``` {.python .jupyter-code-cell}
-pattern = re.compile(r'(.*?)``` {.python .jupyter-code-cell}\n\n(from IPython.display import display, HTML\nhtml_code = """\n.*?\n"""\ndisplay\(HTML\(html_code\)\))\n```(.*)', re.DOTALL)
+pattern = re.compile(r'(.*?)``` {\.python \.jupyter-code-cell}\n(.*?from IPython\.display import display, HTML.*?display\(HTML\(html_code\)\))\n```(.*)', re.DOTALL)
 
 
 def process_video_cell(notebook_path):

.jenkins/validate_tutorials_built.py

@@ -31,6 +31,7 @@
     "prototype_source/vmap_recipe",
     "prototype_source/torchscript_freezing",
     "prototype_source/nestedtensor",
+    "prototype_source/gpu_direct_storage", # requires specific filesystem + GPUDirect Storage to be set up
     "recipes_source/recipes/saving_and_loading_models_for_inference",
     "recipes_source/recipes/saving_multiple_models_in_one_file",
     "recipes_source/recipes/tensorboard_with_pytorch",
@@ -50,6 +51,9 @@
     "intermediate_source/flask_rest_api_tutorial",
     "intermediate_source/text_to_speech_with_torchaudio",
     "intermediate_source/tensorboard_profiler_tutorial", # reenable after 2.0 release.
+    "advanced_source/semi_structured_sparse", # reenable after 3303 is fixed.
+    "intermediate_source/torchrec_intro_tutorial", # reenable after 3302 is fixe
+    "intermediate_source/memory_format_tutorial", # causes other tutorials like torch_logs fail. "state" issue, reseting dynamo didn't help
 ]
 
 def tutorial_source_dirs() -> List[Path]:

README.md

@@ -7,7 +7,7 @@ All the tutorials are now presented as sphinx style documentation at:
 
 # Asking a question
 
-If you hve a qestion about a tutorial, post in https://dev-discuss.pytorch.org/ rather than creating an issue in this repo. Your question will be answered much faster on the dev-discuss forum.
+If you have a question about a tutorial, post in https://dev-discuss.pytorch.org/ rather than creating an issue in this repo. Your question will be answered much faster on the dev-discuss forum.
 
 # Submitting an issue