Test 6

Signed-off-by: Jing Chen <[email protected]>

Author: jgchn

.github/workflows/run-examples-jing.yml

@@ -99,5 +99,5 @@ jobs:
         git config --local user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
         git status
         git add tests/results/
-        git diff --cached --quiet || git commit -m "GH Action updated results file when running examples $(date)"
+        git diff --cached --quiet || git commit -m "github-actions[bot]: Updated results file when running examples on $(date)"
         git push

.github/workflows/run-examples.yml

@@ -64,6 +64,8 @@ jobs:
 
     # Run tests
     - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.head_ref }}
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
@@ -93,4 +95,12 @@ jobs:
         WATSONX_URL: ${{ secrets.WATSONX_URL }}
         REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}
         OLLAMA_GHACTIONS_RESULTS: true
-      run: py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py
+      run: py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py
+    - name: Update example result files (if any) generated from Ollama running on GH Actions
+      run: |
+        git config --local user.name github-actions[bot]
+        git config --local user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
+        git status
+        git add tests/results/
+        git diff --cached --quiet || git commit -m "github-actions[bot]: Updated results file when running examples on $(date)"
+        git push

tests/test_examples_run.py

@@ -16,7 +16,7 @@
 # test_examples_run.py runs the examples and compares the results
 # to the expected results in tests/results/examples
 
-UPDATE_RESULTS = True
+UPDATE_RESULTS = False
 RESULTS_VERSION = 1
 OLLAMA_GHACTIONS_RESULTS_ENV_VAR = os.getenv("OLLAMA_GHACTIONS_RESULTS", "")
 OLLAMA_GHACTIONS_RESULTS = False
@@ -191,28 +191,34 @@ def test_valid_programs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -
                 output="all",
                 config=pdl.InterpreterConfig(batch=0),
             )
-            result = output["result"]
+            actual_result = output["result"]
 
             block_to_dict(output["trace"], json_compatible=True)
             result_dir_name = (
                 pathlib.Path(".") / "tests" / "results" / pdl_file_name.parent
             )
 
-            if not __find_and_compare_results(pdl_file_name, str(result)):
+            if not __find_and_compare_results(pdl_file_name, str(actual_result)):
 
                 if OLLAMA_GHACTIONS_RESULTS:
                     print(
-                        "-------------------- Updating result from running Ollama on GitHub Actions -------------------- "
+                        f"Program {str(pdl_file_name)} requries updating its result on GitHub Actions"
                     )
+                    print(f"Actual results: {str(actual_result)}")
                     result_file_name = f"{pdl_file_name.stem}.ollama_ghactions.result"
                     __write_to_results_file(
-                        result_dir_name, result_file_name, str(result)
+                        result_dir_name, result_file_name, str(actual_result)
                     )
 
                     # Evaluate the results again. If fails again, then consider this program as failing
-                    if not __find_and_compare_results(pdl_file_name, str(result)):
+                    if not __find_and_compare_results(
+                        pdl_file_name, str(actual_result)
+                    ):
+                        print(
+                            f"Program {str(pdl_file_name)} failed second time even after generating results from Github Actions. Consider this failing!"
+                        )
                         wrong_results[str(pdl_file_name)] = {
-                            "actual": str(result),
+                            "actual": str(actual_result),
                         }
                     # If evaluating results produces correct result, then this is considered passing
                     else:
@@ -223,11 +229,11 @@ def test_valid_programs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -
                         f"{pdl_file_name.stem}.{str(RESULTS_VERSION)}.result"
                     )
                     __write_to_results_file(
-                        result_dir_name, result_file_name, str(result)
+                        result_dir_name, result_file_name, str(actual_result)
                     )
 
                 wrong_results[str(pdl_file_name)] = {
-                    "actual": str(result),
+                    "actual": str(actual_result),
                 }
         except PDLParseError:
             actual_parse_error |= {str(pdl_file_name)}

tests/test_examples_run_jing.py

@@ -16,7 +16,7 @@
 # test_examples_run.py runs the examples and compares the results
 # to the expected results in tests/results/examples
 
-UPDATE_RESULTS = True
+UPDATE_RESULTS = False
 RESULTS_VERSION = 1
 OLLAMA_GHACTIONS_RESULTS_ENV_VAR = os.getenv("OLLAMA_GHACTIONS_RESULTS", "")
 OLLAMA_GHACTIONS_RESULTS = False