diff --git a/extension/android/executorch_android/android_test_setup.sh b/extension/android/executorch_android/android_test_setup.sh
index 68f4d9dd6f6..fff8ba33618 100644
--- a/extension/android/executorch_android/android_test_setup.sh
+++ b/extension/android/executorch_android/android_test_setup.sh
@@ -12,17 +12,18 @@ if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
 fi
 which "${PYTHON_EXECUTABLE}"
 
-BASEDIR=$(dirname "$0")
+BASEDIR=$(dirname "$(realpath $0)")
 cp "${BASEDIR}/../../../extension/module/test/resources/add.pte" "${BASEDIR}/src/androidTest/resources"
 
 pushd "${BASEDIR}/../../../"
-curl -Ls "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" --output stories110M.pt
-curl -Ls "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" --output tokenizer.model
+curl -C - -Ls "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" --output stories110M.pt
+curl -C - -Ls "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" --output tokenizer.model
 # Create params.json file
 touch params.json
 echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
-python -m examples.models.llama.export_llama -c stories110M.pt -p params.json -X -kv --model=stories110m
+python -m examples.models.llama.export_llama -c stories110M.pt -p params.json -d fp16 -n stories110m_h.pte -kv
+python -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 
-cp *.pte "${BASEDIR}/src/androidTest/resources/stories.pte"
-cp tokenizer.model "${BASEDIR}/src/androidTest/resources/tokenizer.bin"
+cp stories110m_h.pte "${BASEDIR}/src/androidTest/resources/stories.pte"
+cp tokenizer.bin "${BASEDIR}/src/androidTest/resources/tokenizer.bin"
 popd
diff --git a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
index ae81a21f420..e95b42f2650 100644
--- a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
+++ b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
@@ -86,7 +86,6 @@ public void testGenerate() throws IOException, URISyntaxException{
 
     @Test
     public void testGenerateAndStop() throws IOException, URISyntaxException{
-        int seqLen = 32;
         mModule.generate(TEST_PROMPT, SEQ_LEN, new LlmCallback() {
             @Override
             public void onResult(String result) {
diff --git a/scripts/run_android_emulator.sh b/scripts/run_android_emulator.sh
index fe73ec8a1d7..43ecfc1671a 100755
--- a/scripts/run_android_emulator.sh
+++ b/scripts/run_android_emulator.sh
@@ -26,9 +26,17 @@ adb install -t app-debug-androidTest.apk
 adb shell mkdir -p /data/local/tmp/llama
 adb push model.pte /data/local/tmp/llama
 adb push tokenizer.bin /data/local/tmp/llama
-adb shell am instrument -w -r com.example.executorchllamademo.test/androidx.test.runner.AndroidJUnitRunner
+adb logcat -c
+adb shell am instrument -w -r com.example.executorchllamademo.test/androidx.test.runner.AndroidJUnitRunner >result.txt 2>&1
+adb logcat -d > logcat.txt
+cat logcat.txt
+grep -q FAILURES result.txt || cat result.txt
 
 adb uninstall org.pytorch.executorch.test || true
 adb install -t android-test-debug-androidTest.apk
 
-adb shell am instrument -w -r org.pytorch.executorch.test/androidx.test.runner.AndroidJUnitRunner
+adb logcat -c
+adb shell am instrument -w -r org.pytorch.executorch.test/androidx.test.runner.AndroidJUnitRunner >result.txt 2>&1
+adb logcat -d > logcat.txt
+cat logcat.txt
+grep -q FAILURES result.txt || cat result.txt