Merge branch 'pytorch:main' into main

Author: sekyondaMeta

.jenkins/metadata.json

@@ -61,6 +61,9 @@
   "recipes_source/torch_compile_user_defined_triton_kernel_tutorial.py": {
     "needs": "linux.g5.4xlarge.nvidia.gpu"
   },
+  "recipes_source/regional_compilation.py": {
+    "needs": "linux.g5.4xlarge.nvidia.gpu"
+  },
   "advanced_source/semi_structured_sparse.py": {
     "needs": "linux.g5.4xlarge.nvidia.gpu"
   },

_static/img/python_extension_autoload_impl.png

@@ -57,8 +57,8 @@
 ########################
 # We can then load the model as demonstrated below.
 #
-# As described in `Saving and loading torch.nn.Modules <pytorch.org/docs/main/notes/serialization.html#saving-and-loading-torch-nn-modules>`__,
-# saving ``state_dict``s is considered the best practice. However,
+# As described in `Saving and loading torch.nn.Modules <https://pytorch.org/docs/main/notes/serialization.html#saving-and-loading-torch-nn-modules>`_,
+# saving ``state_dict`` is considered the best practice. However,
 # below we use ``weights_only=False`` because this involves loading the
 # model, which is a legacy use case for ``torch.save``.
 

beginner_source/basics/saveloadrun_tutorial.py

@@ -311,9 +311,7 @@ def forward(self, sentence):
 # ``TransformerDecoder``) and subcomponents (``TransformerEncoderLayer``,
 # ``TransformerDecoderLayer``). For details, check out the
 # `documentation <https://pytorch.org/docs/stable/nn.html#transformer-layers>`__
-# on transformer classes, and the relevant
-# `tutorial <https://pytorch.org/tutorials/beginner/transformer_tutorial.html>`__
-# on pytorch.org.
+# on transformer classes.
 # 
 # Other Layers and Functions
 # --------------------------

beginner_source/introyt/modelsyt_tutorial.py

@@ -439,6 +439,13 @@ Welcome to PyTorch Tutorials
    :link: advanced/python_custom_ops.html
    :tags: Extending-PyTorch,Frontend-APIs,C++,CUDA
 
+.. customcarditem::
+   :header: Compiled Autograd: Capturing a larger backward graph for ``torch.compile``
+   :card_description: Learn how to use compiled autograd to capture a larger backward graph.
+   :image: _static/img/thumbnails/cropped/generic-pytorch-logo.png
+   :link: intermediate/compiled_autograd_tutorial
+   :tags: Model-Optimization,CUDA
+
 .. customcarditem::
    :header: Custom C++ and CUDA Operators
    :card_description: How to extend PyTorch with custom C++ and CUDA operators.
@@ -1132,6 +1139,7 @@ Additional Resources
    intermediate/nvfuser_intro_tutorial
    intermediate/ax_multiobjective_nas_tutorial
    intermediate/torch_compile_tutorial
+   intermediate/compiled_autograd_tutorial
    intermediate/inductor_debug_cpu
    intermediate/scaled_dot_product_attention_tutorial
    beginner/knowledge_distillation_tutorial

index.rst

@@ -244,7 +244,7 @@ def generate_rand_batch(
 
 ######################################################################
 # Using SDPA with ``torch.compile``
-# =================================
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 #
 # With the release of PyTorch 2.0, a new feature called
 # ``torch.compile()`` has been introduced, which can provide
@@ -324,9 +324,9 @@ def generate_rand_batch(
 #
 
 ######################################################################
-# Using SDPA with attn_bias subclasses`
-# ==========================================
-#
+# Using SDPA with attn_bias subclasses
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 # As of PyTorch 2.3, we have added a new submodule that contains tensor subclasses.
 # Designed to be used with ``torch.nn.functional.scaled_dot_product_attention``.
 # The module is named ``torch.nn.attention.bias`` and contains the following two
@@ -394,7 +394,7 @@ def generate_rand_batch(
 
 ######################################################################
 # Conclusion
-# ==========
+# ~~~~~~~~~~~
 #
 # In this tutorial, we have demonstrated the basic usage of
 # ``torch.nn.functional.scaled_dot_product_attention``. We have shown how

intermediate_source/compiled_autograd_tutorial.rst

@@ -917,7 +917,7 @@ def _wait_impl(self) -> torch.Tensor:
 
 ######################################################################
 # Inference
-# ~~~~~~~~~~~~
+# ~~~~~~~~~
 # 
 # Now that we are able to train distributed embeddings, how can we take
 # the trained model and optimize it for inference? Inference is typically

intermediate_source/scaled_dot_product_attention_tutorial.py

@@ -39,3 +39,7 @@ Prototype Tutorials
 10 flight_recorder_tutorial.rst
 	   Flight Recorder User Guide
 	   https://pytorch.org/tutorials/prototype/flight_recorder_tutorial.html
+
+11 python_extension_autoload.rst
+	   Autoloading Out-of-Tree Extension
+	   https://pytorch.org/tutorials/prototype/python_extension_autoload.html

intermediate_source/torchrec_intro_tutorial.py

@@ -9,7 +9,10 @@ What you will learn
 
 Prerequisites
 -------------
+
 - PyTorch version 2.5 or later.
+- `tabulate <https://pypi.org/project/tabulate/>`__. You can install by running ``pip install tabulate``.
+
 
 Overview
 --------