Merge branch 'pytorch:main' into patch-13

Author: mikekgfb

.github/workflows/pull.yml

@@ -731,6 +731,7 @@ jobs:
 
           git clone https://github.com/ggerganov/llama.cpp.git
           pushd llama.cpp
+          git checkout 64ed2091b24b2f9747148fdf49a34ed5938762c3
           make
           popd
 
@@ -941,7 +942,7 @@ jobs:
           path: |
             ./et-build
             ./torchchat/utils/scripts
-          key: et-build-${{runner.os}}-${{runner.arch}}-${{env.et-git-hash}}-${{ hashFiles('**/install_et.sh') }}
+          key: et-build-${{runner.os}}-${{runner.arch}}-${{env.et-git-hash}}-${{ hashFiles('**/install_et.sh', '**/build_native.sh') }}
       - if: ${{ steps.install-et.outputs.cache-hit != 'true' }}
         continue-on-error: true
         run: |
@@ -1052,7 +1053,7 @@ jobs:
 
           # Pull submodules (re2, abseil) for Tiktoken
           git submodule sync
-          git submodule update --init
+          git submodule update --init --recursive
           ./runner/build_android.sh
           echo "Tests complete."
 

.gitmodules

@@ -1,9 +1,3 @@
-[submodule "tokenizer/third-party/abseil-cpp"]
-	path = tokenizer/third-party/abseil-cpp
-	url = https://github.com/abseil/abseil-cpp.git
-[submodule "tokenizer/third-party/re2"]
-	path = tokenizer/third-party/re2
-	url = https://github.com/google/re2.git
-[submodule "tokenizer/third-party/sentencepiece"]
-	path = tokenizer/third-party/sentencepiece
-	url = https://github.com/google/sentencepiece.git
+[submodule "runner/third-party/tokenizers"]
+	path = runner/third-party/tokenizers
+	url = https://github.com/pytorch-labs/tokenizers

CMakeLists.txt

@@ -7,18 +7,21 @@ ELSE()
 ENDIF()
 
 project(Torchchat)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
 
 # include tokenizer
-add_subdirectory(tokenizer)
+add_subdirectory(runner/third-party/tokenizers)
 
 # include et_run executable
 include(runner/et.cmake)
 if(TARGET et_run)
-    target_link_libraries(et_run PUBLIC tokenizer microkernels-prod)
+    target_link_libraries(et_run PUBLIC tokenizers microkernels-prod)
+    target_include_directories(et_run PUBLIC runner/third-party/tokenizers/include)
 endif()
 
 # include aoti_run executable
 include(runner/aoti.cmake)
 if(TARGET aoti_run)
-    target_link_libraries(aoti_run tokenizer)
+    target_link_libraries(aoti_run tokenizers)
+    target_include_directories(aoti_run PUBLIC runner/third-party/tokenizers/include)
 endif()

docs/quantization.md

@@ -59,7 +59,7 @@ for valid `bitwidth` and `groupsize` values.
 | linear with dynamic activations (symmetric) | `'{"linear:a8w4dq" : {"groupsize" : <groupsize>}}'`|
 | embedding | `'{"embedding": {"bitwidth": <bitwidth>, "groupsize":<groupsize>}}'` |
 
-See the available quantization schemes [here](https://github.com/pytorch/torchchat/blob/main/torchchat/utils/quantize.py#L1260-L1266).
+See the available quantization schemes [here](https://github.com/pytorch/torchchat/blob/b809b69e03f8f4b75a4b27b0778f0d3695ce94c2/torchchat/utils/quantize.py#L887-L894).
 
 In addition to quantization, the [accelerator](model_customization.md#device)
 and [precision](model_customization.md#model-precision) can also be specified.

install/.pins/et-pin.txt

@@ -1 +1 @@
-72b3bb3194c611f7c4861e6f3b24af5de868af72
+98e4dd524f2cb08414ee015b27616229cabc06ba

install/install_requirements.sh

@@ -62,10 +62,10 @@ echo "Using pip executable: $PIP_EXECUTABLE"
 # NOTE: If a newly-fetched version of the executorch repo changes the value of
 # PYTORCH_NIGHTLY_VERSION, you should re-run this script to install the necessary
 # package versions.
-PYTORCH_NIGHTLY_VERSION=dev20241002
+PYTORCH_NIGHTLY_VERSION=dev20241010
 
 # Nightly version for torchvision
-VISION_NIGHTLY_VERSION=dev20241002
+VISION_NIGHTLY_VERSION=dev20241010
 
 # Nightly version for torchtune
 TUNE_NIGHTLY_VERSION=dev20241010