Run unit test in test-huggingface-transformers job

Author: larryliu0820

.github/workflows/_unittest.yml

@@ -30,25 +30,19 @@ jobs:
     permissions:
       id-token: write
       contents: read
-    secrets: inherit
     with:
-      secrets-env: EXECUTORCH_HF_TOKEN
       runner: linux.2xlarge
       docker-image: ${{ inputs.docker-image }}
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       timeout: 90
       script: |
         set -eux
-        pip install -U "huggingface_hub[cli]"
-        huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
         .ci/scripts/unittest-linux.sh --build-tool "${{ inputs.build-tool }}" --build-mode "${{ inputs.build-mode }}" --editable "${{ inputs.editable }}"
 
   macos:
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
-    secrets: inherit
     with:
-      secrets-env: EXECUTORCH_HF_TOKEN
       runner: macos-m1-stable
       python-version: '3.11'
       submodules: 'recursive'
@@ -57,6 +51,4 @@ jobs:
         set -eux
         # This is needed to get the prebuilt PyTorch wheel from S3
         ${CONDA_RUN} --no-capture-output pip install awscli==1.37.21
-        ${CONDA_RUN} pip install -U "huggingface_hub[cli]"
-        huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
         .ci/scripts/unittest-macos.sh --build-tool "${{ inputs.build-tool }}" --build-mode "${{ inputs.build-mode }}" --editable "${{ inputs.editable }}"

pytest.ini

@@ -65,7 +65,6 @@ addopts =
     extension/llm/custom_ops/test_sdpa_with_kv_cache.py
     extension/llm/custom_ops/test_update_cache.py
     extension/llm/custom_ops/test_quantized_sdpa.py
-    extension/llm/optimum/test
     extension/pybindings/test
     extension/training/pybindings/test
     # Runtime