[Windows] Re-enable tokenizers

Author: GregoryComer

.ci/scripts/setup-windows.ps1

@@ -13,6 +13,11 @@ conda activate et
 # Install test dependencies
 pip install -r .ci/docker/requirements-ci.txt
 
+# Create a symlink to work around path length issues when building submodules (tokenizers).
+Push-Location
+New-Item -ItemType SymbolicLink -Path "C:\_et" -Target "$CWD"
+cd C:\_et
+
 if ($editable -eq 'true') {
     install_executorch.bat --editable
 } else {
@@ -22,3 +27,5 @@ if ($LASTEXITCODE -ne 0) {
     Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE."
     exit $LASTEXITCODE
 }
+
+Pop-Location

install_requirements.py

@@ -112,13 +112,8 @@ def install_requirements(use_pytorch_nightly):
 
     LOCAL_REQUIREMENTS = [
         "third-party/ao",  # We need the latest kernels for fast iteration, so not relying on pypi.
-    ] + (
-        [
-            "extension/llm/tokenizers",  # TODO(larryliu0820): Setup a pypi package for this.
-        ]
-        if sys.platform != "win32"
-        else []
-    )  # TODO(gjcomer): Re-enable when buildable on Windows.
+        "extension/llm/tokenizers",  # TODO(larryliu0820): Setup a pypi package for this.
+    ]
 
     # Install packages directly from local copy instead of pypi.
     # This is usually not recommended.