From 2ffa89a4d2f440af62c9bd852b522b4f909d7853 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sat, 12 Apr 2025 16:35:41 -0400 Subject: [PATCH 01/37] Pr pipeline 2 (#5) * Cleaning up run examples pipeline Signed-off-by: Jing Chen Rm run examples composite Signed-off-by: Jing Chen Halt early if no file patch Signed-off-by: Jing Chen Better msg Signed-off-by: Jing Chen Modify some PDL files Signed-off-by: Jing Chen Revert revisions Signed-off-by: Jing Chen Modify some PDL programs Signed-off-by: Jing Chen Fix if Signed-off-by: Jing Chen Composite run examples Signed-off-by: Jing Chen Checkout before using composite Test if Test if again Rm bash test Fix unfound path Update if check Delete a folder and see if test runs Bring back deleted example. Expects test to run Test if examples config is patched Mv yaml patch to composite action Try another patch strategy Change some files Checkout again in run examples Modify files Typo in composite action Use json in check parse Try to escape json Use changes instead in yaml action Bring back Ollama set up Gather new results from GH env Clean up workflows Fix lint Modify chatbot example Fix run examples config file * Add possible results for react examples * Preserve trailing white space * See if a commit is pushed on my behalf * Rename config file * Diff pdl file * Check if commit happens despite pytet failures * Allow pytest failure and continue run * github-actions[bot]: Updated results file when running examples on Mon Apr 7 23:24:42 UTC 2025 Signed-off-by: github-actions[bot] <75508867+jgchn@users.noreply.github.com> * Set max parallel to 1 to prevent race conditions * See if yaml action is triggered * Change master branch * Fix pytest * Set update results to true * Test yaml actio again * Update result to true * Reverting update_results to False. If you wish to update again, change locally and push with change. * WRite permissions * Reverting update_results to False. If you wish to update again, change locally and push with change. * Don't commit using yaml action * Set update results to true * github-actions[bot]: Updated results file when running examples on Tue Apr 8 20:11:04 UTC 2025 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Reverting update_results to False. Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Try pytest failure * Test pytest behavior again * Test set +e * Test again * Set update to true to see if changed-files can capture results * PR test * github-actions[bot]: Updated results file when running examples Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * github-actions[bot]: Reverting update_result to false Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Test only with PR, not push * github-actions[bot]: Updated results file when running examples Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * github-actions[bot]: Reverting update_result to false Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * No pdl file change * Clean up and fix run examples nightly Signed-off-by: Jing Chen * Fix empty parse * Fix * Bring back more models on ollama set up * Contrib readme updates --------- Signed-off-by: github-actions[bot] <75508867+jgchn@users.noreply.github.com> Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Jing Chen Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: github-actions[bot] <75508867+github-actions[bot]jgchn@users.noreply.github.com> Get new results for failing progs github-actions[bot]: Run examples: updated result files on your behalf Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> github-actions[bot]: Run examples: reverting update_result to false on your behalf Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> See if ollama is cached Try caching ollama Set up Ollama outside composite action Revert changes Update runexamples config Signed-off-by: Jing Chen --- .github/actions/ollama/action.yml | 48 ++ .github/actions/run-examples/action.yml | 103 ++++ .github/workflows/run-examples-prep.yml | 90 ++++ .github/workflows/run-examples.yml | 100 +--- docs/contrib.md | 51 +- examples/chatbot/chatbot.pdl | 2 +- examples/demo/9-react.pdl | 1 + examples/react/demo.pdl | 1 + ...lama_ghactions.result => chatbot.1.result} | 0 ...ma_ghactions.result => code-eval.1.result} | 0 .../results/examples/code/code-eval.2.result | 27 + ...ma_ghactions.result => code-json.1.result} | 0 ....ollama_ghactions.result => code.1.result} | 0 tests/results/examples/code/code.2.result | 21 + ...llama_ghactions.result => 10-sdg.1.result} | 0 ...ma_ghactions.result => 3-def-use.1.result} | 0 ...a_ghactions.result => 4-function.1.result} | 0 ..._ghactions.result => 5-code-eval.1.result} | 0 ..._ghactions.result => 6-code-json.1.result} | 0 ...ctions.result => 7-chatbot-roles.1.result} | 0 tests/results/examples/demo/9-react.1.result | 19 + tests/results/examples/demo/9-react.2.result | 26 + tests/results/examples/demo/9-react.3.result | 24 + ...b.ollama_ghactions.result => fib.1.result} | 0 tests/results/examples/react/demo.1.result | 18 + tests/results/examples/react/demo.2.result | 25 + tests/results/examples/react/demo.3.result | 18 + ...lama_ghactions.result => teacher.1.result} | 0 ...ling_llm_with_input_messages_var.1.result} | 0 ....ollama_ghactions.result => defs.1.result} | 0 ...ns.result => function_definition.1.result} | 0 ...result => function_empty_context.1.result} | 0 .../examples/tutorial/input_stdin.0.result | 2 +- ...ions.result => parser_regex_code.1.result} | 0 ...lama_ghactions.result => chatbot.1.result} | 0 ...ma_ghactions.result => code-json.1.result} | 0 ...lama_ghactions.result => weather.1.result} | 0 tests/test_examples_run.py | 505 ++++++++++-------- tests/test_examples_run.yaml | 97 ++++ 39 files changed, 859 insertions(+), 319 deletions(-) create mode 100644 .github/actions/ollama/action.yml create mode 100644 .github/actions/run-examples/action.yml create mode 100644 .github/workflows/run-examples-prep.yml rename tests/results/examples/chatbot/{chatbot.ollama_ghactions.result => chatbot.1.result} (100%) rename tests/results/examples/code/{code-eval.ollama_ghactions.result => code-eval.1.result} (100%) create mode 100644 tests/results/examples/code/code-eval.2.result rename tests/results/examples/code/{code-json.ollama_ghactions.result => code-json.1.result} (100%) rename tests/results/examples/code/{code.ollama_ghactions.result => code.1.result} (100%) create mode 100644 tests/results/examples/code/code.2.result rename tests/results/examples/demo/{10-sdg.ollama_ghactions.result => 10-sdg.1.result} (100%) rename tests/results/examples/demo/{3-def-use.ollama_ghactions.result => 3-def-use.1.result} (100%) rename tests/results/examples/demo/{4-function.ollama_ghactions.result => 4-function.1.result} (100%) rename tests/results/examples/demo/{5-code-eval.ollama_ghactions.result => 5-code-eval.1.result} (100%) rename tests/results/examples/demo/{6-code-json.ollama_ghactions.result => 6-code-json.1.result} (100%) rename tests/results/examples/demo/{7-chatbot-roles.ollama_ghactions.result => 7-chatbot-roles.1.result} (100%) create mode 100644 tests/results/examples/demo/9-react.1.result create mode 100644 tests/results/examples/demo/9-react.2.result create mode 100644 tests/results/examples/demo/9-react.3.result rename tests/results/examples/fibonacci/{fib.ollama_ghactions.result => fib.1.result} (100%) create mode 100644 tests/results/examples/react/demo.1.result create mode 100644 tests/results/examples/react/demo.2.result create mode 100644 tests/results/examples/react/demo.3.result rename tests/results/examples/teacher/{teacher.ollama_ghactions.result => teacher.1.result} (100%) rename tests/results/examples/tutorial/{calling_llm_with_input_messages_var.ollama_ghactions.result => calling_llm_with_input_messages_var.1.result} (100%) rename tests/results/examples/tutorial/{defs.ollama_ghactions.result => defs.1.result} (100%) rename tests/results/examples/tutorial/{function_definition.ollama_ghactions.result => function_definition.1.result} (100%) rename tests/results/examples/tutorial/{function_empty_context.ollama_ghactions.result => function_empty_context.1.result} (100%) rename tests/results/examples/tutorial/{parser_regex_code.ollama_ghactions.result => parser_regex_code.1.result} (100%) rename tests/results/examples/tutorial/programs/{chatbot.ollama_ghactions.result => chatbot.1.result} (100%) rename tests/results/examples/tutorial/programs/{code-json.ollama_ghactions.result => code-json.1.result} (100%) rename tests/results/examples/tutorial/programs/{weather.ollama_ghactions.result => weather.1.result} (100%) create mode 100644 tests/test_examples_run.yaml diff --git a/.github/actions/ollama/action.yml b/.github/actions/ollama/action.yml new file mode 100644 index 000000000..47e6d7094 --- /dev/null +++ b/.github/actions/ollama/action.yml @@ -0,0 +1,48 @@ +name: 'Ollama Setup' +description: 'Composte action for Ollama set up in GH environment' +runs: + using: 'composite' + steps: + - name: Remove unnecessary files + shell: bash + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + # Set up Ollama + - name: Install Ollama and start server + shell: bash + run: | + curl -fsSL https://ollama.com/install.sh | sudo -E sh + + - name: Pull models in examples/ + shell: bash + run: | + ollama pull granite3.2:2b + ollama pull granite3.2:8b + ollama pull mxbai-embed-large + ollama list + + - name: Check that all required models are available + shell: bash + run: | + models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b") + missing=0 + for model in "${models[@]}"; do + if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then + echo "❌ Model $model is missing!" + missing=1 + fi + done + + if [ "$missing" -eq 1 ]; then + exit 1 + else + echo "✅ All expected models are available." + fi + + - name: Wait for Ollama server + shell: bash + run: | + sleep 5 + time curl -i http://localhost:11434 \ No newline at end of file diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml new file mode 100644 index 000000000..e370c69d7 --- /dev/null +++ b/.github/actions/run-examples/action.yml @@ -0,0 +1,103 @@ +name: 'Ollama Setup' +description: 'Composte action to set up Run Examples' +inputs: + python-version: + description: 'Python version' + required: true + default: '3.11' + runner-os: + description: 'Runner OS' + required: true + head-ref: + description: 'Head ref of the repo' + required: true + update-results: + description: 'Whether to update the results for this run. Must be false for nightly runs' + required: true + check: + description: 'Files to patch tests/test_examples_run.yaml with. These are the PDL files that the test will run against. Defaults to all PDL files.' + required: false + default: '[]' +runs: + using: 'composite' + steps: + # # Set up Ollama + - uses: ./.github/actions/ollama + + # Configure Run Examples environment + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.head-ref }} + fetch-depth: 0 # Full history needed for fetch/rebase + - name: Patch tests/test_examples_run.yaml check with modified files + uses: fjogeleit/yaml-update-action@main + with: + changes: | + { + "tests/test_examples_run.yaml": { + "check": ${{ inputs.check }} + } + } + commitChange: false + - name: Print test Run Examples config + shell: bash + run: cat tests/test_examples_run.yaml + + # Run tests + - name: Set up Python ${{ inputs.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Cache pip + uses: actions/cache@v4 + with: + # This path is specific to Ubuntu + path: ${{ env.pythonLocation }} + # Look to see if there is a cache hit for the setup file + key: ${{ inputs.runner-os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ inputs.runner-os }}-pip-new3 + ${{ inputs.runner-os }}-new3 + - name: Install dependencies + shell: bash + run: pip install --upgrade --upgrade-strategy eager .[all] + - name: Pip list packages + shell: bash + run: pip list + - name: Run Pytest + shell: bash + run: | + cat tests/test_examples_run.yaml + ( + set +e + py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings + EXIT_CODE=$? + + if [ $EXIT_CODE -eq 0 ]; then + echo "TEST_RESULT=PASSED" >> $GITHUB_ENV + else + echo "TEST_RESULT=FAILED" >> $GITHUB_ENV + fi + ) + + # Commit the results if update results + - name: Push new results to branch + shell: bash + if: ${{ inputs.update-results == 'true' }} + run: | + git config --local user.name github-actions[bot] + git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com + git status + git pull + git add tests/results/ + git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" + git push + + - name: Check if pytest passed + shell: bash + run: | + if [ "$TEST_RESULT" == "PASSED" ]; then + exit 0 + else + exit 1 + fi diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml new file mode 100644 index 000000000..ff2290a3c --- /dev/null +++ b/.github/workflows/run-examples-prep.yml @@ -0,0 +1,90 @@ +--- +name: Run examples on modified PDL files +on: [pull_request] +jobs: + tests: + name: Execution tests + runs-on: ubuntu-latest + permissions: + contents: write + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python-version: ['3.11', '3.12', '3.13'] + steps: + # Detect modified PDL files, includes Add, Modified, but not Deleted + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Detect all PDL files that were changed or added + id: changed-pdl-files + uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46 + with: + files: | + **.pdl + json: "true" + escape_json: "false" + - name: List PDL files that were modified or added and append to test_examples_run + env: + MODIFIED_PDL_FILES: ${{ steps.changed-pdl-files.outputs.all_changed_files }} + run: echo "$MODIFIED_PDL_FILES" + + # Run tests if there are modified PDL files + - uses: actions/checkout@v4 + if: ${{ steps.changed-pdl-files.outputs.all_changed_files_count > 0 }} + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + + - name: Check if update results automatically by GitHub Actions Bot + id: parse-run-examples-config + shell: bash + run: | + value=$(yq '.update_results' tests/test_examples_run.yaml | tr '[:upper:]' '[:lower:]') + echo "value=$value" + echo "update_results=$value" >> $GITHUB_OUTPUT + + - uses: ./.github/actions/run-examples + if: ${{ steps.changed-pdl-files.outputs.all_changed_files_count > 0 }} + with: + python-version: ${{ matrix.python-version }} + runner-os: ${{ runner.os }} + update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} + check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} + head-ref: ${{ github.head_ref }} + + # Patch config with update_results to false + # Fetch the latest branch + - uses: actions/checkout@v4 + if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} + with: + ref: ${{ github.head_ref }} + fetch-depth: 0 + - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version + if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} + uses: fjogeleit/yaml-update-action@main + with: + changes: | + { + "tests/test_examples_run.yaml": { + "update_results": false + } + } + commitChange: false + - name: Push the new run-examples yaml config + if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} + shell: bash + run: | + git config --local user.name github-actions[bot] + git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com + git status + git pull + git add tests/test_examples_run.yaml + git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: reverting update_result to false on your behalf" + git push + + + + + \ No newline at end of file diff --git a/.github/workflows/run-examples.yml b/.github/workflows/run-examples.yml index f3e75674b..6cc92c3d3 100644 --- a/.github/workflows/run-examples.yml +++ b/.github/workflows/run-examples.yml @@ -6,7 +6,6 @@ on: - cron: '0 1 * * *' workflow_dispatch: - jobs: tests: name: Execution tests @@ -15,93 +14,14 @@ jobs: fail-fast: false matrix: python-version: ['3.11', '3.12', '3.13'] - steps: - - # Free up some disk space - - name: Remove unnecessary files - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - # Set up Ollama - - name: Install Ollama and start server - shell: bash - run: | - curl -fsSL https://ollama.com/install.sh | sudo -E sh - - - name: Pull models in examples/ - shell: bash - run: | - ollama pull granite3.2:2b - ollama pull granite3.2:8b - ollama pull mxbai-embed-large - ollama list - - - name: Check that all required models are available - shell: bash - run: | - models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b") - missing=0 - for model in "${models[@]}"; do - if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then - echo "❌ Model $model (or substring) is missing!" - missing=1 - fi - done - - if [ "$missing" -eq 1 ]; then - exit 1 - else - echo "✅ All expected models are available." - fi - - - name: Wait for Ollama server - shell: bash - run: | - sleep 10 - time curl -i http://localhost:11434 - - # Run tests - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v4 - with: - # This path is specific to Ubuntu - path: ${{ env.pythonLocation }} - # Look to see if there is a cache hit for the setup file - key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - restore-keys: | - ${{ runner.os }}-pip-new3 - ${{ runner.os }}-new3 - - name: Install dependencies - run: pip install --upgrade --upgrade-strategy eager .[all] - - name: pip list packages - run: pip list - - name: show pip dependencies - run: | - pip install pipdeptree - pipdeptree -fl - - name: run tests - env: - WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} - WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }} - WATSONX_URL: ${{ secrets.WATSONX_URL }} - REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }} - OLLAMA_GHACTIONS_RESULTS: true - run: py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py - - name: Update example result files (if any) generated from Ollama running on GH Actions - if: matrix.python-version == '3.11' - run: | - git config --local user.name github-actions[bot] - git config --local user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com" - git status - git add tests/results/ - git diff --cached --quiet || git commit -S -s -m "github-actions[bot]: Updated results file when running examples on $(date)" - git push \ No newline at end of file + - uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + - uses: ./.github/actions/run-examples + with: + python-version: ${{ matrix.python-version }} + runner-os: ${{ runner.os }} + head-ref: ${{ github.head_ref }} + update-results: 'false' + check: '[]' # Empty list means run against all PDL programs diff --git a/docs/contrib.md b/docs/contrib.md index e4995b0ea..bf1fa7996 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -61,8 +61,7 @@ When you make changes to PDL, ensure to document any new features in the docs se Install the required dependencies for documentation. ``` -pip install mkdocs-get-deps -pip install $(mkdocs-get-deps) +pip install -e .[docs] ``` Then serve the docs to load a preview. @@ -71,4 +70,50 @@ Then serve the docs to load a preview. mkdocs serve ``` -You are all set! \ No newline at end of file +You are all set! + +### Run examples + +PDL executes nightly runs for Run Examples, which searches for all the `.pdl` programs in the repo and runs the interpreter against each file. The [config file for Run Examples](../tests/test_examples_run.yaml) describes how to handle with each file. There are four conditions: + +1. `skip`: a list of PDL files that are skipped in Run Examples +2. `with_inputs`: PDL files that require user input. Each file name is mapped to two fields that describe how inputs are patched + 1. `stdin`: separated by a newline, each line represents the user input + 2. `scope`: scope for the PDL program +3. `expected_parse_error`: a list of PDL files that expect parse errors +4. `expected_runtime_error`: a list of PDL files that expect runtime errors + +If you wish to make a contribution to PDL and modify or add any PDL program to the repo, it is important that you provide the new expected results for those files so that the Run Examples nightly test does not break. + +### Local dev + +Under `check`, you can provide a list of files that you want to run Pytest against. If you leave it empty (`check: ` or `check: []`), then by default, Pytest will be executed against all files in the repo, except for those under `skip`. For local development, it is useful to only test against a subset of files so that Pytest executes faster. + +If you expect the files to produce a different result, setting `update_results: true` will automatically create a new file under `tests/examples/results` capturing the new output for each of the file in `check`. It is useful to set this field to `true` before opening a PR. + +Run this Pytest command for Run Examples, which is the same command for the nightly test. + +``` +pytest --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings +``` + +### Opening a pull request + +A slight variation in the Python version and OS environment can cause a different LLM response, thus Run Examples might fail because it uses exact string matching for PDL outputs. You can utilize the `update_results` when opening a PR to capture the expected results in the GH Actions environment. + +When you open a pull request (PR) against the `main` branch, a series of status checks will be executed. Specificially, three Run Examples test will be initiated against the files you have added and modified. + +- If in the PR, `update_results: false`, Pytest will be executed against those files + - If Pytest fails, you can manually examine the failures and add the results to your PR +- If in the PR, `update_results: true`, Pytest will be executed against those files. + - If Pytest fails, a GH-Actions bot will push a commit with the new results produced in the GH environment to your PR + - Additionally, another commit will push `update_results: false`. + - If you need to make other changes to your PR, make sure to pull the new changes first. + +Your PR should always set `update_results: false` before merging. + +Here's a preview of the current configuration file for Run Examples: + +```yaml +--8<-- "../tests/test_examples_run.yaml" +``` diff --git a/examples/chatbot/chatbot.pdl b/examples/chatbot/chatbot.pdl index a53f68efd..3c89de4b9 100644 --- a/examples/chatbot/chatbot.pdl +++ b/examples/chatbot/chatbot.pdl @@ -1,4 +1,4 @@ -description: Chatbot +description: A Chatbot that responds to user's questions text: # Allow the user to type any question, implicitly adding the question to the context. - read: diff --git a/examples/demo/9-react.pdl b/examples/demo/9-react.pdl index 80eccde60..dd5742e12 100644 --- a/examples/demo/9-react.pdl +++ b/examples/demo/9-react.pdl @@ -1,3 +1,4 @@ +description: Demonstrating a React program defs: tools: data: diff --git a/examples/react/demo.pdl b/examples/react/demo.pdl index f50210e2e..fdfbfe2a2 100644 --- a/examples/react/demo.pdl +++ b/examples/react/demo.pdl @@ -1,3 +1,4 @@ +description: Demo for a react pattern program defs: tools: data: diff --git a/tests/results/examples/chatbot/chatbot.ollama_ghactions.result b/tests/results/examples/chatbot/chatbot.1.result similarity index 100% rename from tests/results/examples/chatbot/chatbot.ollama_ghactions.result rename to tests/results/examples/chatbot/chatbot.1.result diff --git a/tests/results/examples/code/code-eval.ollama_ghactions.result b/tests/results/examples/code/code-eval.1.result similarity index 100% rename from tests/results/examples/code/code-eval.ollama_ghactions.result rename to tests/results/examples/code/code-eval.1.result diff --git a/tests/results/examples/code/code-eval.2.result b/tests/results/examples/code/code-eval.2.result new file mode 100644 index 000000000..31fa397a7 --- /dev/null +++ b/tests/results/examples/code/code-eval.2.result @@ -0,0 +1,27 @@ +@SuppressWarnings("unchecked") +public static Map deserializeOffsetMap(String lastSourceOffset) throws IOException { + Map offsetMap; + if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { + offsetMap = new HashMap<>(); + } else { + offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); + } + return offsetMap; +} +The provided Java function `deserializeOffsetMap` is part of the StreamSets Data Collector (datacollector) project, specifically located in the `stagesupport/src/main/java/com/` directory. This function is named `OffsetUtil.deserializeOffsetMap`. Here's a breakdown of its purpose and functionality: +1. **Purpose**: The primary goal of this method is to deserialize a JSON string into a `Map` object representing an offset map. Offsets are typically used in data processing pipelines to track the position within input data streams. +2. **Parameters**: + - `lastSourceOffset`: A required parameter that represents the JSON-encoded offset map as a string. This is the source of the deserialization process. +3. **Return Type**: The method returns a `Map`. This means it will return a map where keys are strings and values are also strings. +4. **Code Explanation**: + - `@SuppressWarnings("unchecked")`: This annotation tells the compiler to ignore potential warnings related to unchecked casts. In this context, it's safe because we're working with `Map` objects, which are known to be compatible with `java.lang.Object`. + + - `public static Map deserializeOffsetMap(String lastSourceOffset) throws IOException`: This is the method signature. It declares a public static method named `deserializeOffsetMap`, which takes one parameter (a string representing JSON-encoded offset map data), and returns a `Map`. The method also specifies that it may throw an `IOException`. + - `Map offsetMap;`: This line initializes a variable called `offsetMap` to hold the deserialized map. + - `if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { ... } else { ... }`: The method uses a conditional statement to check if the input string is either `null` or empty. If so, it creates and initializes an empty `HashMap` as `offsetMap`. + - `offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class);`: When the condition in the previous line evaluates to false (i.e., when `lastSourceOffset` is not null or empty), this line performs the actual deserialization using Jackson's `JSON_MAPPER`. It reads the input string as a JSON object and converts it into a `Map`. + - `return offsetMap;`: Finally, after processing the input string (either creating an empty map if necessary or performing the JSON-to-map conversion), this line returns the resulting `offsetMap` to the caller. +In summary, `deserializeOffsetMap` is a utility function that safely converts a JSON-encoded offset map string into a `Map`. It handles both null and empty inputs gracefully by initializing an empty map when necessary before performing the actual deserialization using Jackson's JSON mapper. +EVALUATION: +The similarity (Levenshtein) between this answer and the ground truth is: +0.1734802701741912 \ No newline at end of file diff --git a/tests/results/examples/code/code-json.ollama_ghactions.result b/tests/results/examples/code/code-json.1.result similarity index 100% rename from tests/results/examples/code/code-json.ollama_ghactions.result rename to tests/results/examples/code/code-json.1.result diff --git a/tests/results/examples/code/code.ollama_ghactions.result b/tests/results/examples/code/code.1.result similarity index 100% rename from tests/results/examples/code/code.ollama_ghactions.result rename to tests/results/examples/code/code.1.result diff --git a/tests/results/examples/code/code.2.result b/tests/results/examples/code/code.2.result new file mode 100644 index 000000000..2bd33a88c --- /dev/null +++ b/tests/results/examples/code/code.2.result @@ -0,0 +1,21 @@ +@SuppressWarnings("unchecked") +public static Map deserializeOffsetMap(String lastSourceOffset) throws IOException { + Map offsetMap; + if (lastSourceOffset == null || lastSourceOffset.isEmpty()) { + offsetMap = new HashMap<>(); + } else { + offsetMap = JSON_MAPPER.readValue(lastSourceOffset, Map.class); + } + return offsetMap; +} +The provided Java function `deserializeOffsetMap` is part of the StreamSets Data Collector (datacollector) repository, specifically located in the `stagesupport/src/main/java/com/` directory. This function is named `OffsetUtil.deserializeOffsetMap`. Here's a breakdown of its purpose and functionality: +1. **Purpose**: The primary goal of this method is to deserialize a JSON string into a `Map` object. It assumes that the input JSON represents an offset map with keys as strings (e.g., "record_id", "timestamp") and values also being strings (e.g., "1234567890" or "2022-01-01T00:00:00Z"). +2. **Input**: The method takes a single parameter, `lastSourceOffset`, which is expected to be a JSON string representing an offset map. If this input is null or empty (i.e., `null` or `""`), the function initializes and returns a new `HashMap` with no entries. +3. **Deserialization**: + - When `lastSourceOffset` is not null or empty: + - The method uses Jackson's `JSON_MAPPER`, an instance of `ObjectMapper`, to parse the JSON string into a `Map`. This is done using the `readValue()` method with `Map.class` as the target class. + - If `lastSourceOffset` is null or empty: + - The function initializes and returns a new `HashMap` with no entries (i.e., an empty map). +4. **Return Value**: Regardless of whether the input was null or non-empty, this method always returns a `Map`. This ensures that the caller can safely use the returned object without worrying about potential null values. +5. **Exception Handling**: The function does not explicitly handle `IOException`. However, since it's called within the context of StreamSets Data Collector (datacollector), any underlying I/O issues are likely to be managed by the framework itself. +In summary, this method serves as a utility for converting JSON strings into Map objects representing offset data. It ensures that null or empty inputs result in an empty map, while non-empty inputs are parsed using Jackson's `JSON_MAPPER`. \ No newline at end of file diff --git a/tests/results/examples/demo/10-sdg.ollama_ghactions.result b/tests/results/examples/demo/10-sdg.1.result similarity index 100% rename from tests/results/examples/demo/10-sdg.ollama_ghactions.result rename to tests/results/examples/demo/10-sdg.1.result diff --git a/tests/results/examples/demo/3-def-use.ollama_ghactions.result b/tests/results/examples/demo/3-def-use.1.result similarity index 100% rename from tests/results/examples/demo/3-def-use.ollama_ghactions.result rename to tests/results/examples/demo/3-def-use.1.result diff --git a/tests/results/examples/demo/4-function.ollama_ghactions.result b/tests/results/examples/demo/4-function.1.result similarity index 100% rename from tests/results/examples/demo/4-function.ollama_ghactions.result rename to tests/results/examples/demo/4-function.1.result diff --git a/tests/results/examples/demo/5-code-eval.ollama_ghactions.result b/tests/results/examples/demo/5-code-eval.1.result similarity index 100% rename from tests/results/examples/demo/5-code-eval.ollama_ghactions.result rename to tests/results/examples/demo/5-code-eval.1.result diff --git a/tests/results/examples/demo/6-code-json.ollama_ghactions.result b/tests/results/examples/demo/6-code-json.1.result similarity index 100% rename from tests/results/examples/demo/6-code-json.ollama_ghactions.result rename to tests/results/examples/demo/6-code-json.1.result diff --git a/tests/results/examples/demo/7-chatbot-roles.ollama_ghactions.result b/tests/results/examples/demo/7-chatbot-roles.1.result similarity index 100% rename from tests/results/examples/demo/7-chatbot-roles.ollama_ghactions.result rename to tests/results/examples/demo/7-chatbot-roles.1.result diff --git a/tests/results/examples/demo/9-react.1.result b/tests/results/examples/demo/9-react.1.result new file mode 100644 index 000000000..c00d32058 --- /dev/null +++ b/tests/results/examples/demo/9-react.1.result @@ -0,0 +1,19 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Apologies for the confusion, but it seems there's no specific information available about Henry Hudson's exact birthdate. However, he is estimated to have been born around 1565. +Thought: Since I can't find an exact birthdate, I'll use the approximate year and calculate based on that. +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: \ No newline at end of file diff --git a/tests/results/examples/demo/9-react.2.result b/tests/results/examples/demo/9-react.2.result new file mode 100644 index 000000000..14c95d61f --- /dev/null +++ b/tests/results/examples/demo/9-react.2.result @@ -0,0 +1,26 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. + +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Apologies for the confusion, but it seems there's no specific information available about Henry Hudson's exact birthdate. However, he is estimated to have been born around 1565. + + +Thought: Since I can't find an exact birthdate, I'll use the approximate year and calculate based on that. + + +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: \ No newline at end of file diff --git a/tests/results/examples/demo/9-react.3.result b/tests/results/examples/demo/9-react.3.result new file mode 100644 index 000000000..a670f1757 --- /dev/null +++ b/tests/results/examples/demo/9-react.3.result @@ -0,0 +1,24 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. + +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Apologies for the confusion, but it seems there's no specific information available about Henry Hudson's exact birthdate. However, he is estimated to have been born around 1565. + + +Thought: Since I can't find an exact birthdate, I'll use the approximate year and calculate based on that. + + +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: diff --git a/tests/results/examples/fibonacci/fib.ollama_ghactions.result b/tests/results/examples/fibonacci/fib.1.result similarity index 100% rename from tests/results/examples/fibonacci/fib.ollama_ghactions.result rename to tests/results/examples/fibonacci/fib.1.result diff --git a/tests/results/examples/react/demo.1.result b/tests/results/examples/react/demo.1.result new file mode 100644 index 000000000..2b4214546 --- /dev/null +++ b/tests/results/examples/react/demo.1.result @@ -0,0 +1,18 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Apologies for the confusion, but it seems there's no specific information available about Henry Hudson's exact birthdate. However, he is estimated to have been born around 1565. +Thought: Since I can't find an exact birthdate, I'll use the approximate year and calculate based on that. +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] \ No newline at end of file diff --git a/tests/results/examples/react/demo.2.result b/tests/results/examples/react/demo.2.result new file mode 100644 index 000000000..f7748937e --- /dev/null +++ b/tests/results/examples/react/demo.2.result @@ -0,0 +1,25 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. + +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Apologies for the confusion, but it seems there's no specific information available about Henry Hudson's exact birthdate. However, he is estimated to have been born around 1565. + + +Thought: Since I can't find an exact birthdate, I'll use the approximate year and calculate based on that. + + +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] \ No newline at end of file diff --git a/tests/results/examples/react/demo.3.result b/tests/results/examples/react/demo.3.result new file mode 100644 index 000000000..440e8fcee --- /dev/null +++ b/tests/results/examples/react/demo.3.result @@ -0,0 +1,18 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Thought: I couldn't find a specific birthdate for Henry Hudson, but it's estimated he was born around 1565. + +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Thought: Henry Hudson was born around 1565, so that's approximately 460 years ago.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] diff --git a/tests/results/examples/teacher/teacher.ollama_ghactions.result b/tests/results/examples/teacher/teacher.1.result similarity index 100% rename from tests/results/examples/teacher/teacher.ollama_ghactions.result rename to tests/results/examples/teacher/teacher.1.result diff --git a/tests/results/examples/tutorial/calling_llm_with_input_messages_var.ollama_ghactions.result b/tests/results/examples/tutorial/calling_llm_with_input_messages_var.1.result similarity index 100% rename from tests/results/examples/tutorial/calling_llm_with_input_messages_var.ollama_ghactions.result rename to tests/results/examples/tutorial/calling_llm_with_input_messages_var.1.result diff --git a/tests/results/examples/tutorial/defs.ollama_ghactions.result b/tests/results/examples/tutorial/defs.1.result similarity index 100% rename from tests/results/examples/tutorial/defs.ollama_ghactions.result rename to tests/results/examples/tutorial/defs.1.result diff --git a/tests/results/examples/tutorial/function_definition.ollama_ghactions.result b/tests/results/examples/tutorial/function_definition.1.result similarity index 100% rename from tests/results/examples/tutorial/function_definition.ollama_ghactions.result rename to tests/results/examples/tutorial/function_definition.1.result diff --git a/tests/results/examples/tutorial/function_empty_context.ollama_ghactions.result b/tests/results/examples/tutorial/function_empty_context.1.result similarity index 100% rename from tests/results/examples/tutorial/function_empty_context.ollama_ghactions.result rename to tests/results/examples/tutorial/function_empty_context.1.result diff --git a/tests/results/examples/tutorial/input_stdin.0.result b/tests/results/examples/tutorial/input_stdin.0.result index 4270aa50d..d3f68b23e 100644 --- a/tests/results/examples/tutorial/input_stdin.0.result +++ b/tests/results/examples/tutorial/input_stdin.0.result @@ -1,2 +1,2 @@ The following will prompt the user on stdin. -Hello \ No newline at end of file +What is APR? \ No newline at end of file diff --git a/tests/results/examples/tutorial/parser_regex_code.ollama_ghactions.result b/tests/results/examples/tutorial/parser_regex_code.1.result similarity index 100% rename from tests/results/examples/tutorial/parser_regex_code.ollama_ghactions.result rename to tests/results/examples/tutorial/parser_regex_code.1.result diff --git a/tests/results/examples/tutorial/programs/chatbot.ollama_ghactions.result b/tests/results/examples/tutorial/programs/chatbot.1.result similarity index 100% rename from tests/results/examples/tutorial/programs/chatbot.ollama_ghactions.result rename to tests/results/examples/tutorial/programs/chatbot.1.result diff --git a/tests/results/examples/tutorial/programs/code-json.ollama_ghactions.result b/tests/results/examples/tutorial/programs/code-json.1.result similarity index 100% rename from tests/results/examples/tutorial/programs/code-json.ollama_ghactions.result rename to tests/results/examples/tutorial/programs/code-json.1.result diff --git a/tests/results/examples/tutorial/programs/weather.ollama_ghactions.result b/tests/results/examples/tutorial/programs/weather.1.result similarity index 100% rename from tests/results/examples/tutorial/programs/weather.ollama_ghactions.result rename to tests/results/examples/tutorial/programs/weather.1.result diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index d3e08689a..47646d777 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -2,272 +2,329 @@ import os import pathlib import random -from dataclasses import dataclass -from typing import Optional +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, List, Optional +import yaml from pytest import CaptureFixture, MonkeyPatch from pdl import pdl from pdl.pdl_ast import ScopeType -from pdl.pdl_dumper import block_to_dict from pdl.pdl_lazy import PdlDict from pdl.pdl_parser import PDLParseError -# test_examples_run.py runs the examples and compares the results -# to the expected results in tests/results/examples - -UPDATE_RESULTS = False -RESULTS_VERSION = 1 -OLLAMA_GHACTIONS_RESULTS_ENV_VAR = os.getenv("OLLAMA_GHACTIONS_RESULTS", "") -OLLAMA_GHACTIONS_RESULTS = False -if OLLAMA_GHACTIONS_RESULTS_ENV_VAR.lower().strip() == "true": - OLLAMA_GHACTIONS_RESULTS = True - -TO_SKIP = { - str(name) - for name in [ - # Requires dataset dependency - pathlib.Path("examples") / "cldk" / "cldk-assistant.pdl", - pathlib.Path("examples") / "gsm8k" / "gsm8.pdl", - pathlib.Path("examples") / "gsm8k" / "gsm8k-plan.pdl", - # Requires installation dependencies - pathlib.Path("examples") / "intrinsics" / "demo-hallucination.pdl", - pathlib.Path("examples") / "tutorial" / "programs" / "demo-hallucination.pdl", - # Skip RAG examples - pathlib.Path("examples") / "rag" / "pdf_index.pdl", - pathlib.Path("examples") / "rag" / "pdf_query.pdl", - pathlib.Path("examples") - / "rag" - / "rag_library1.pdl", # (This is glue to Python, it doesn't "run" alone) - # Skip structure decoding example (Jing doesn't have WATSONX API KEY) - pathlib.Path("examples") / "tutorial" / "structured_decoding.pdl", - # OUtput result include trace (and thus timing) for some reason. Investigate why - pathlib.Path("examples") / "react" / "react_call.pdl", - pathlib.Path("pdl-live-react") / "demos" / "error.pdl", - pathlib.Path("pdl-live-react") / "demos" / "demo1.pdl", - pathlib.Path("pdl-live-react") / "demos" / "demo2.pdl", - # For now, skip the granite-io examples - pathlib.Path("examples") / "granite-io" / "granite_io_hallucinations.pdl", - pathlib.Path("examples") / "granite-io" / "granite_io_openai.pdl", - pathlib.Path("examples") / "granite-io" / "granite_io_thinking.pdl", - pathlib.Path("examples") / "granite-io" / "granite_io_transformers.pdl", - ] -} +EXAMPLES_RUN_CONFIG_FILE = os.getenv( + "EXAMPLES_RUN_FILE", "tests/test_examples_run.yaml" +) @dataclass class InputsType: + """ + Inputs to the PDL program for testing + """ + stdin: Optional[str] = None scope: Optional[ScopeType] = None -TESTS_WITH_INPUT: dict[str, InputsType] = { - str(name): inputs - for name, inputs in { - pathlib.Path("examples") - / "tutorial" - / "programs" - / "chatbot.pdl": InputsType(stdin="What is APR?\nyes\n"), - pathlib.Path("examples") - / "tutorial" - / "input_stdin.pdl": InputsType(stdin="Hello\n"), - pathlib.Path("examples") - / "tutorial" - / "input_stdin_multiline.pdl": InputsType(stdin="Hello\nBye\n"), - pathlib.Path("examples") - / "input" - / "input_test1.pdl": InputsType(stdin="Hello\n"), - pathlib.Path("examples") - / "input" - / "input_test2.pdl": InputsType(stdin="Hello\n"), - pathlib.Path("examples") - / "chatbot" - / "chatbot.pdl": InputsType(stdin="What is APR?\nyes\n"), - pathlib.Path("examples") - / "demo" - / "7-chatbot-roles.pdl": InputsType(stdin="What is APR?\nquit\n"), - pathlib.Path("examples") - / "tutorial" - / "free_variables.pdl": InputsType(scope=PdlDict({"something": "ABC"})), - }.items() -} - - -EXPECTED_PARSE_ERROR = [ - pathlib.Path("tests") / "data" / "line" / "hello.pdl", - pathlib.Path("tests") / "data" / "line" / "hello1.pdl", - pathlib.Path("tests") / "data" / "line" / "hello4.pdl", - pathlib.Path("tests") / "data" / "line" / "hello7.pdl", - pathlib.Path("tests") / "data" / "line" / "hello8.pdl", - pathlib.Path("tests") / "data" / "line" / "hello10.pdl", - pathlib.Path("tests") / "data" / "line" / "hello11.pdl", - pathlib.Path("tests") / "data" / "line" / "hello31.pdl", -] - -EXPECTED_RUNTIME_ERROR = [ - pathlib.Path("examples") / "callback" / "repair_prompt.pdl", - pathlib.Path("examples") / "tutorial" / "type_list.pdl", - pathlib.Path("examples") / "tutorial" / "type_checking.pdl", - pathlib.Path("tests") / "data" / "line" / "hello12.pdl", - pathlib.Path("tests") / "data" / "line" / "hello13.pdl", - pathlib.Path("tests") / "data" / "line" / "hello14.pdl", - pathlib.Path("tests") / "data" / "line" / "hello15.pdl", - pathlib.Path("tests") / "data" / "line" / "hello16.pdl", - pathlib.Path("tests") / "data" / "line" / "hello17.pdl", - pathlib.Path("tests") / "data" / "line" / "hello18.pdl", - pathlib.Path("tests") / "data" / "line" / "hello19.pdl", - pathlib.Path("tests") / "data" / "line" / "hello20.pdl", - pathlib.Path("tests") / "data" / "line" / "hello21.pdl", - pathlib.Path("tests") / "data" / "line" / "hello22.pdl", - pathlib.Path("tests") / "data" / "line" / "hello23.pdl", - pathlib.Path("tests") / "data" / "line" / "hello24.pdl", - pathlib.Path("tests") / "data" / "line" / "hello25.pdl", - pathlib.Path("tests") / "data" / "line" / "hello26.pdl", - pathlib.Path("tests") / "data" / "line" / "hello27.pdl", - pathlib.Path("tests") / "data" / "line" / "hello28.pdl", - pathlib.Path("tests") / "data" / "line" / "hello29.pdl", - pathlib.Path("tests") / "data" / "line" / "hello3.pdl", - pathlib.Path("tests") / "data" / "line" / "hello30.pdl", - pathlib.Path("tests") / "data" / "line" / "hello9.pdl", -] - - -def __write_to_results_file( - dir_name: pathlib.Path, filename: str, content: str -) -> None: +class ExecutionErrorCode(Enum): """ - Write to results file + PDLExecutionErrorCode describes the execution result of the PDL file """ - dir_name.mkdir(parents=True, exist_ok=True) - with open(dir_name / filename, "w", encoding="utf-8") as result_file: - result_file.write(content) + NO_ERROR = 1 + PARSE_ERROR = 2 + RUNTIME_ERROR = 3 -def __find_and_compare_results( - test_file_name: pathlib.Path, actual_result: str -) -> bool: +@dataclass +class ExecutionResult: """ - Look through test_file_name's parent directory and see if any of *.result - matches the actual output + ExecutionResult captures the execution result of a PDL file """ - result_dir_name = pathlib.Path(".") / "tests" / "results" / test_file_name.parent - expected_files = result_dir_name.glob(test_file_name.stem + ".*.result") + result: str | None = None + error_code: ExecutionErrorCode | None = None + + +@dataclass +class ExpectedResult: + """ + ExpectedResult captures the expected result of a PDL file. + Non-deterministic programs contain more than one valid result + """ + + results: List[str] | None = None + error_code: ExecutionErrorCode | None = None + + def compare_to_execution(self, execution_result: ExecutionResult) -> bool: + """ + Returns true if execution matches to expected results and false otherwise + """ + + # ExecutionErrorCode codes must match + if execution_result.error_code != self.error_code: + return False + + # Check if parse or runtime error + if self.error_code == ExecutionErrorCode.PARSE_ERROR: + return execution_result.error_code == ExecutionErrorCode.PARSE_ERROR + if self.error_code == ExecutionErrorCode.RUNTIME_ERROR: + return execution_result.error_code == ExecutionErrorCode.RUNTIME_ERROR + + # At this point, it's NO_ERROR, so check for results + actual_result = execution_result.result + if actual_result is not None and self.results is not None: + actual_result_stripped = actual_result.strip() + for expected_result in self.results: + expected_result_stripped = expected_result.strip() + if actual_result_stripped == expected_result_stripped: + return True + return False + + def get_next_results_version(self) -> int: + """ + Returns the next results version for this file + """ + + if self.results is None: + return 0 + return len(self.results) + + +@dataclass +class FailedResults: + """ + FailedResults are all the files that failed + """ + + wrong_results: Dict[str, str] = field(default_factory=lambda: {}) + unexpected_parse_error: List[str] = field(default_factory=lambda: []) + unexpected_runtime_error: List[str] = field(default_factory=lambda: []) + + +# pylint: disable=too-many-instance-attributes +class ExamplesRun: + """ + ExamplesRun outlines PDL files + - to skip + - requires inputs + - expects parse error + - expects runtime error + by loading the configuration from EXAMPLES_RUN_FILE + and runs the test + """ + + def __init__(self, monkeypatch: MonkeyPatch) -> None: + # Pytest + self.monkeypatch = monkeypatch + + # Configuration + self.update_results: bool = False + + # File manipulation + self.check = [str(f) for f in pathlib.Path(".").glob("**/*.pdl")] + self.skip: List[str] = [] + self.with_inputs: Dict[str, InputsType] = {} + self.expected_parse_error: List[str] = [] + self.expected_runtime_error: List[str] = [] + + # Load content from EXAMPLES_RUN_FILE + with open(EXAMPLES_RUN_CONFIG_FILE, "r", encoding="utf-8") as file: + content = yaml.safe_load(file) + self.update_results = content["update_results"] + + # Update files to check iff check is specified + if content["check"] is not None: + if len(content["check"]) > 0: + self.check = content["check"] + + self.skip = content["skip"] + self.expected_parse_error = content["expected_parse_error"] + self.expected_runtime_error = content["expected_runtime_error"] + + for filename, inputs_type in content["with_inputs"].items(): + stdin = inputs_type["stdin"] + scope = inputs_type["scope"] + self.with_inputs[filename] = InputsType( + stdin=stdin, scope=PdlDict(scope) if scope is not None else None + ) + + # Inits expected results + self.expected_results: Dict[str, ExpectedResult] = {} + self.__collect_expected_results() - for expected_file in expected_files: - with open(expected_file, "r", encoding="utf-8") as truth_file: - expected_result = str(truth_file.read()) - if str(actual_result).strip() == expected_result.strip(): - return True - return False + # Inits execution results for each PDL file + self.execution_results: Dict[str, ExecutionResult] = {} + # Init failed results + self.failed_results = FailedResults() -def test_valid_programs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> None: - actual_parse_error: set[str] = set() - actual_runtime_error: set[str] = set() - wrong_results = {} + def __get_results_dir(self) -> pathlib.Path: + return pathlib.Path(".") / "tests" / "results" - files = pathlib.Path(".").glob("**/*.pdl") + def __collect_expected_results(self) -> None: + """ + Collects possible results for programs in self.check + """ - for pdl_file_name in files: + for file in self.check: + expected_result = ExpectedResult() + if file in self.expected_parse_error: + expected_result.error_code = ExecutionErrorCode.PARSE_ERROR + elif file in self.expected_runtime_error: + expected_result.error_code = ExecutionErrorCode.RUNTIME_ERROR + else: + + # Collect possible results + res_list = [] + file_path: pathlib.Path = pathlib.Path(file) + result_dir_name = self.__get_results_dir() / file_path.parent + expected_files = result_dir_name.glob(file_path.stem + ".*.result") + + for expected_file in expected_files: + with open(expected_file, "r", encoding="utf-8") as truth_file: + content = truth_file.read() + res_list.append(content) + + expected_result.error_code = ExecutionErrorCode.NO_ERROR + expected_result.results = res_list + + self.expected_results[file] = expected_result + + def __execute_file(self, pdl_file_name: str) -> None: + """ + Tests the result of a single file and returns the result output and the error code + """ + + exec_result = ExecutionResult() + + pdl_file_path = pathlib.Path(pdl_file_name) scope: ScopeType = PdlDict({}) - if str(pdl_file_name) in TO_SKIP: - continue - if str(pdl_file_name) in TESTS_WITH_INPUT: - inputs = TESTS_WITH_INPUT[str(pdl_file_name)] + + # Patch with inputs + if pdl_file_name in self.with_inputs: + inputs = self.with_inputs[pdl_file_name] if inputs.stdin is not None: - monkeypatch.setattr( - "sys.stdin", - io.StringIO(inputs.stdin), - ) + self.monkeypatch.setattr("sys.stdin", io.StringIO(inputs.stdin)) if inputs.scope is not None: scope = inputs.scope + try: - random.seed(11) + # Execute file output = pdl.exec_file( - pdl_file_name, + pdl_file_path, scope=scope, output="all", config=pdl.InterpreterConfig(batch=0), ) - actual_result = output["result"] - block_to_dict(output["trace"], json_compatible=True) - result_dir_name = ( - pathlib.Path(".") / "tests" / "results" / pdl_file_name.parent - ) + exec_result.result = str(output["result"]) + exec_result.error_code = ExecutionErrorCode.NO_ERROR - if not __find_and_compare_results(pdl_file_name, str(actual_result)): - - if OLLAMA_GHACTIONS_RESULTS: - print( - f"Program {str(pdl_file_name)} requries updating its result on GitHub Actions" - ) - print(f"Actual results: {str(actual_result)}") - result_file_name = f"{pdl_file_name.stem}.ollama_ghactions.result" - __write_to_results_file( - result_dir_name, result_file_name, str(actual_result) - ) - - # Evaluate the results again. If fails again, then consider this program as failing - if not __find_and_compare_results( - pdl_file_name, str(actual_result) - ): - print( - f"Program {str(pdl_file_name)} failed second time even after generating results from Github Actions. Consider this failing!" - ) - wrong_results[str(pdl_file_name)] = { - "actual": str(actual_result), - } - # If evaluating results produces correct result, then this is considered passing - else: - continue - - if UPDATE_RESULTS: - result_file_name = ( - f"{pdl_file_name.stem}.{str(RESULTS_VERSION)}.result" - ) - __write_to_results_file( - result_dir_name, result_file_name, str(actual_result) - ) - - wrong_results[str(pdl_file_name)] = { - "actual": str(actual_result), - } except PDLParseError: - actual_parse_error |= {str(pdl_file_name)} - except Exception as exc: - if str(pdl_file_name) not in set(str(p) for p in EXPECTED_RUNTIME_ERROR): - print(f"{pdl_file_name}: {exc}") # unexpected error: breakpoint - actual_runtime_error |= {str(pdl_file_name)} - print(exc) - - # Parse errors - expected_parse_error = set(str(p) for p in EXPECTED_PARSE_ERROR) - unexpected_parse_error = sorted(list(actual_parse_error - expected_parse_error)) - assert ( - len(unexpected_parse_error) == 0 - ), f"Unexpected parse error: {unexpected_parse_error}" - - # Runtime errors - expected_runtime_error = set(str(p) for p in EXPECTED_RUNTIME_ERROR) - unexpected_runtime_error = sorted( - list(actual_runtime_error - expected_runtime_error) - ) - assert ( - len(unexpected_runtime_error) == 0 - ), f"Unexpected runtime error: {unexpected_runtime_error}" - - # Unexpected valid - unexpected_valid = sorted( - list( - (expected_parse_error - actual_parse_error).union( - expected_runtime_error - actual_runtime_error + exec_result.error_code = ExecutionErrorCode.PARSE_ERROR + except Exception: + exec_result.error_code = ExecutionErrorCode.RUNTIME_ERROR + + self.execution_results[pdl_file_name] = exec_result + + def populate_exec_result_for_checks(self) -> None: + """ + Populates the execution result for all files in self.checks + """ + + for file in self.check: + if file not in self.skip: + self.__execute_file(file) + + def validate_expected_and_actual(self) -> None: + """ + Validates the expected result to actual result + Must be run after populate_exec_result_for_checks + """ + + wrong_result: Dict[str, str] = {} + unexpected_parse_error: List[str] = [] + unexpected_runtime_error: List[str] = [] + + for file in self.check: + if file not in self.skip: + expected_result = self.expected_results[file] + actual_result = self.execution_results[file] + match = expected_result.compare_to_execution(actual_result) + + if not match: + # Check if actual results caused any error + if actual_result.error_code == ExecutionErrorCode.PARSE_ERROR: + unexpected_parse_error.append(file) + elif actual_result.error_code == ExecutionErrorCode.RUNTIME_ERROR: + unexpected_runtime_error.append(file) + # If no error, then the results are wrong + else: + if actual_result.result is not None: + wrong_result[file] = actual_result.result + + self.failed_results.wrong_results = wrong_result + self.failed_results.unexpected_parse_error = unexpected_parse_error + self.failed_results.unexpected_runtime_error = unexpected_runtime_error + + def write_results(self) -> None: + """ + Writes new results for failed files + """ + + results_dir = self.__get_results_dir() + for file in self.failed_results.wrong_results: + next_results_version = str( + self.expected_results[file].get_next_results_version() ) + # Mkdir if not exist + file_path = pathlib.Path(file) + write_file_dir = results_dir / file_path.parent + write_file_dir.mkdir(parents=True, exist_ok=True) + + # Write to new file + write_file_name = ( + write_file_dir / f"{file_path.stem}.{next_results_version}.result" + ) + actual_result = self.failed_results.wrong_results[file] + + with open(write_file_name, "w", encoding="utf-8") as f: + f.write(actual_result) + + +def test_example_runs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> None: + """ + Runs the test + """ + + random.seed(11) + background = ExamplesRun(monkeypatch) + + background.populate_exec_result_for_checks() + background.validate_expected_and_actual() + + if background.update_results: + background.write_results() + + # Print the actual results for wrong results + for file, actual in background.failed_results.wrong_results.items(): + print( + "============================================================================" ) - ) - assert len(unexpected_valid) == 0, f"Unexpected valid: {unexpected_valid}" - # Unexpected results - assert len(wrong_results) == 0, f"Wrong results: {wrong_results}" + print(f"File that produced wrong result: {file}") + print(f"Actual:\n{actual}\n") + + assert ( + len(background.failed_results.unexpected_parse_error) == 0 + ), f"Unexpected parse error: {background.failed_results.unexpected_parse_error}" + assert ( + len(background.failed_results.unexpected_runtime_error) == 0 + ), f"Unexpected runtime error: {background.failed_results.unexpected_runtime_error}" + assert ( + len(background.failed_results.wrong_results) == 0 + ), f"Wrong results: {background.failed_results.wrong_results}" diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml new file mode 100644 index 000000000..81b66d229 --- /dev/null +++ b/tests/test_examples_run.yaml @@ -0,0 +1,97 @@ +update_results: false +check: [] +skip: + - examples/cldk/cldk-assistant.pdl + - examples/gsm8k/gsm8.pdl + - examples/gsm8k/gsm8k-plan.pdl + - examples/gsm8k/gsm8k-tot.pdl + - examples/gsm8k/gsm8k-tot-multiplan.pdl + - examples/gsm8k/gsm8k-tot-few-shot.pdl + - examples/gsm8k/gsm8k-plan-few-shots.pdl + - examples/intrinsics/demo-hallucination.pdl + - examples/tutorial/programs/demo-hallucination.pdl + - examples/rag/pdf_index.pdl + - examples/rag/pdf_query.pdl + - examples/rag/rag_library1.pdl + - examples/tutorial/structured_decoding.pdl + - examples/react/react_call.pdl + - pdl-live-react/demos/error.pdl + - pdl-live-react/demos/demo1.pdl + - pdl-live-react/demos/demo2.pdl + - pdl-live-react/src-tauri/tests/cli/read-stdin.pdl + - examples/granite-io/granite_io_hallucinations.pdl + - examples/granite-io/granite_io_openai.pdl + - examples/granite-io/granite_io_thinking.pdl + - examples/granite-io/granite_io_transformers.pdl +with_inputs: + examples/tutorial/programs/chatbot.pdl: + stdin: | + What is APR? + yes + scope: null + examples/chatbot/chatbot.pdl: + stdin: | + What is APR? + yes + scope: null + examples/demo/7-chatbot-roles.pdl: + stdin: | + What is APR? + quit + scope: null + examples/tutorial/input_stdin.pdl: + stdin: | + What is APR? + yes + scope: null + examples/tutorial/input_stdin_multiline.pdl: + stdin: | + Hello + Bye + scope: null + examples/input/input_test1.pdl: + stdin: | + Hello + scope: null + examples/input/input_test2.pdl: + stdin: | + Hello + scope: null + examples/tutorial/free_variables.pdl: + stdin: null + scope: + something: ABC +expected_parse_error: + - tests/data/line/hello.pdl + - tests/data/line/hello1.pdl + - tests/data/line/hello4.pdl + - tests/data/line/hello7.pdl + - tests/data/line/hello8.pdl + - tests/data/line/hello10.pdl + - tests/data/line/hello11.pdl + - tests/data/line/hello31.pdl +expected_runtime_error: + - examples/callback/repair_prompt.pdl + - examples/tutorial/type_list.pdl + - examples/tutorial/type_checking.pdl + - tests/data/line/hello3.pdl + - tests/data/line/hello9.pdl + - tests/data/line/hello12.pdl + - tests/data/line/hello13.pdl + - tests/data/line/hello14.pdl + - tests/data/line/hello15.pdl + - tests/data/line/hello16.pdl + - tests/data/line/hello17.pdl + - tests/data/line/hello18.pdl + - tests/data/line/hello19.pdl + - tests/data/line/hello20.pdl + - tests/data/line/hello21.pdl + - tests/data/line/hello22.pdl + - tests/data/line/hello23.pdl + - tests/data/line/hello24.pdl + - tests/data/line/hello25.pdl + - tests/data/line/hello26.pdl + - tests/data/line/hello27.pdl + - tests/data/line/hello28.pdl + - tests/data/line/hello29.pdl + - tests/data/line/hello30.pdl From 54b6ab2b4dbe586a1d7fd435a4ae3f314389b14b Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sat, 12 Apr 2025 19:11:52 -0400 Subject: [PATCH 02/37] Update contrib readme Signed-off-by: Jing Chen --- docs/contrib.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/contrib.md b/docs/contrib.md index bf1fa7996..9fb51cc39 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -85,7 +85,7 @@ PDL executes nightly runs for Run Examples, which searches for all the `.pdl` pr If you wish to make a contribution to PDL and modify or add any PDL program to the repo, it is important that you provide the new expected results for those files so that the Run Examples nightly test does not break. -### Local dev +#### Local dev Under `check`, you can provide a list of files that you want to run Pytest against. If you leave it empty (`check: ` or `check: []`), then by default, Pytest will be executed against all files in the repo, except for those under `skip`. For local development, it is useful to only test against a subset of files so that Pytest executes faster. @@ -97,18 +97,18 @@ Run this Pytest command for Run Examples, which is the same command for the nigh pytest --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings ``` -### Opening a pull request +#### Opening a pull request A slight variation in the Python version and OS environment can cause a different LLM response, thus Run Examples might fail because it uses exact string matching for PDL outputs. You can utilize the `update_results` when opening a PR to capture the expected results in the GH Actions environment. When you open a pull request (PR) against the `main` branch, a series of status checks will be executed. Specificially, three Run Examples test will be initiated against the files you have added and modified. -- If in the PR, `update_results: false`, Pytest will be executed against those files +- If in the PR, `update_results: false`, - If Pytest fails, you can manually examine the failures and add the results to your PR -- If in the PR, `update_results: true`, Pytest will be executed against those files. +- If in the PR, `update_results: true`, - If Pytest fails, a GH-Actions bot will push a commit with the new results produced in the GH environment to your PR - - Additionally, another commit will push `update_results: false`. - - If you need to make other changes to your PR, make sure to pull the new changes first. + - Additionally, another commit will push `update_results: false`. This is to ensure that for the nightly test, different results are not committed + - If you need to make other changes to your PR, make sure to pull the new changes first Your PR should always set `update_results: false` before merging. From ec0a03196aa603c17c779a1d1856d05e9b8c8d0e Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sat, 12 Apr 2025 19:34:04 -0400 Subject: [PATCH 03/37] Get test results for nightly run Signed-off-by: Jing Chen --- .github/workflows/run-examples.yml | 2 +- tests/test_examples_run.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-examples.yml b/.github/workflows/run-examples.yml index 6cc92c3d3..7964fb0e3 100644 --- a/.github/workflows/run-examples.yml +++ b/.github/workflows/run-examples.yml @@ -23,5 +23,5 @@ jobs: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} head-ref: ${{ github.head_ref }} - update-results: 'false' + update-results: 'true' check: '[]' # Empty list means run against all PDL programs diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 81b66d229..76426ad28 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,4 +1,4 @@ -update_results: false +update_results: true check: [] skip: - examples/cldk/cldk-assistant.pdl From a745413e78eda50d77bb17778eba19aee15b4d99 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sat, 12 Apr 2025 19:52:46 -0400 Subject: [PATCH 04/37] Update results on all files github-actions[bot]: Run examples: reverting update_result to false on your behalf Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> github-actions[bot]: Run examples: updated result files on your behalf Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Revert patches Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 6 +++-- .github/workflows/run-examples.yml | 2 +- tests/results/examples/react/demo.4.result | 23 +++++++++++++++++++ .../src-tauri/tests/cli/call-no-args.0.result | 1 + .../tests/cli/call-with-args.0.result | 1 + .../src-tauri/tests/cli/code-python.0.result | 1 + .../src-tauri/tests/cli/data1.0.result | 1 + .../src-tauri/tests/cli/data2.0.result | 1 + .../src-tauri/tests/cli/data3.0.result | 1 + .../src-tauri/tests/cli/data4.0.result | 1 + .../src-tauri/tests/cli/if1.0.result | 1 + .../src-tauri/tests/cli/if2.0.result | 1 + .../src-tauri/tests/cli/include1.0.result | 1 + .../tests/cli/json-parser-lastOf.0.result | 1 + .../src-tauri/tests/cli/json-parser.0.result | 1 + .../tests/cli/model-input-array.0.result | 1 + .../tests/cli/model-input-nested.0.result | 5 ++++ .../tests/cli/model-input-string.0.result | 12 ++++++++++ .../src-tauri/tests/cli/object1.0.result | 1 + .../src-tauri/tests/cli/object2.0.result | 1 + .../src-tauri/tests/cli/read-file.0.result | 1 + .../src-tauri/tests/cli/repeat1.0.result | 1 + .../src-tauri/tests/cli/repeat2.0.result | 1 + .../src-tauri/tests/cli/repeat3.0.result | 1 + .../src-tauri/tests/cli/scoping_1.0.result | 1 + .../tests/cli/scoping_1_wrapper.0.result | 1 + tests/test_examples_run.yaml | 2 +- 27 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 tests/results/examples/react/demo.4.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/call-no-args.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/call-with-args.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/code-python.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/data1.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/data2.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/data3.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/data4.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/if1.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/if2.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/include1.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/json-parser-lastOf.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/json-parser.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/model-input-array.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/model-input-nested.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/model-input-string.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/object1.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/object2.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/read-file.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/repeat1.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/repeat2.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/repeat3.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1.0.result create mode 100644 tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1_wrapper.0.result diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index ff2290a3c..e7197a1a3 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -50,8 +50,10 @@ jobs: with: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} - update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} - check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} + # update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} + # check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} + update-results: 'true' + check: '[]' head-ref: ${{ github.head_ref }} # Patch config with update_results to false diff --git a/.github/workflows/run-examples.yml b/.github/workflows/run-examples.yml index 7964fb0e3..6cc92c3d3 100644 --- a/.github/workflows/run-examples.yml +++ b/.github/workflows/run-examples.yml @@ -23,5 +23,5 @@ jobs: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} head-ref: ${{ github.head_ref }} - update-results: 'true' + update-results: 'false' check: '[]' # Empty list means run against all PDL programs diff --git a/tests/results/examples/react/demo.4.result b/tests/results/examples/react/demo.4.result new file mode 100644 index 000000000..eda5d82e1 --- /dev/null +++ b/tests/results/examples/react/demo.4.result @@ -0,0 +1,23 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. + +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Apologies for the confusion, but it seems there's no specific information available about Henry Hudson's exact birthdate. However, he is estimated to have been born around 1565. + + +Thought: Since I can't find an exact birthdate, I'll use the approximate year and calculate based on that. + + +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/call-no-args.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/call-no-args.0.result new file mode 100644 index 000000000..95d09f2b1 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/call-no-args.0.result @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/call-with-args.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/call-with-args.0.result new file mode 100644 index 000000000..b62a2faca --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/call-with-args.0.result @@ -0,0 +1 @@ +hello world 4 bye \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/code-python.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/code-python.0.result new file mode 100644 index 000000000..73f1631b5 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/code-python.0.result @@ -0,0 +1 @@ +{'foo': 3} \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/data1.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/data1.0.result new file mode 100644 index 000000000..d5b8008b3 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/data1.0.result @@ -0,0 +1 @@ +xxxx3True \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/data2.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/data2.0.result new file mode 100644 index 000000000..d5b8008b3 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/data2.0.result @@ -0,0 +1 @@ +xxxx3True \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/data3.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/data3.0.result new file mode 100644 index 000000000..13da5d23b --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/data3.0.result @@ -0,0 +1 @@ +${x}3True \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/data4.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/data4.0.result new file mode 100644 index 000000000..d311d3c43 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/data4.0.result @@ -0,0 +1 @@ +yyyyxxxx3True \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/if1.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/if1.0.result new file mode 100644 index 000000000..32f95c0d1 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/if1.0.result @@ -0,0 +1 @@ +hi \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/if2.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/if2.0.result new file mode 100644 index 000000000..476e93d57 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/if2.0.result @@ -0,0 +1 @@ +good \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/include1.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/include1.0.result new file mode 100644 index 000000000..b62a2faca --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/include1.0.result @@ -0,0 +1 @@ +hello world 4 bye \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/json-parser-lastOf.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/json-parser-lastOf.0.result new file mode 100644 index 000000000..2890eead2 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/json-parser-lastOf.0.result @@ -0,0 +1 @@ +value \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/json-parser.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/json-parser.0.result new file mode 100644 index 000000000..a1d9a8735 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/json-parser.0.result @@ -0,0 +1 @@ +{"key": "value"}value \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-array.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-array.0.result new file mode 100644 index 000000000..b6fe4fd38 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-array.0.result @@ -0,0 +1 @@ +As an assistant living in Europe, I'd be happy to share that the fastest land animal native to this continent is actually the Greyhound. These dogs are renowned for their incredible speed and can reach top speeds of around 43-45 miles per hour (70 km/h). They were originally bred for hunting, particularly for their ability to chase down prey at high velocities. \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-nested.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-nested.0.result new file mode 100644 index 000000000..fae2ece6a --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-nested.0.result @@ -0,0 +1,5 @@ +{"role": "user", "content": "answer as if you live in europe", "defsite": "text.0.message"}{"role": "user", "content": "what is the fastest animal where i live?", "defsite": "text.1.text.0.message"}As an assistant living in Europe, I'd be happy to share that the fastest land animal native to this continent is actually the Greyhound dog breed. These dogs are renowned for their incredible speed and agility on the ground. They can reach top speeds of around 45-48 miles per hour (72-77 kilometers per hour), making them one of the fastest animals in Europe. + +However, if we consider wild animals, the Cheetah is the fastest land animal globally and can be found in parts of Africa, including regions near the European border with North African countries like Libya and Tunisia. Cheetahs typically reach speeds up to 60-70 miles per hour (97-113 kilometers per hour), but they are not native to Europe. + +For a truly European animal, the Greyhound remains the fastest land creature in our region. \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-string.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-string.0.result new file mode 100644 index 000000000..6b71649df --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/model-input-string.0.result @@ -0,0 +1,12 @@ +The fastest animal is the cheetah. This remarkable creature can reach speeds up to 60-70 miles per hour (97-113 kilometers per hour) in short bursts, making it the world's fastest land animal. Cheetahs have several adaptations that contribute to their incredible speed: + +1. **Muscular Build**: Cheetahs have a lean, muscular body with long, slender limbs and a flexible spine. This allows for efficient movement during high-speed chases. +2. **Large Lungs and Heart**: Their lungs are large relative to their size, enabling them to take in more oxygen per breath. A cheetah's heart can beat up to 500 times per minute during intense running, pumping blood rich in oxygen to support its muscles. +3. **Specialized Claws**: Cheetahs have non-retractable claws that provide better grip and traction on the ground while sprinting. +4. **Flexible Spine**: Their spines are flexible, allowing them to maintain a running stride without breaking their backs – unlike other big cats with rigid spines. +5. **Tail**: Cheetahs use their long, bushy tails for balance and steering during high-speed pursuits. +6. **Short, Muscular Legs**: Their legs are short but incredibly powerful due to the muscles attached to them. This helps in generating force for acceleration and maintaining top speed. +7. **Efficient Fat Burning**: Cheetahs have a high proportion of fast-twitch muscle fibers, which burn fat quickly for energy during sprints. +8. **Smooth Skin**: Their skin is smooth to minimize air resistance while running at high speeds. + +Despite their incredible speed, cheetahs cannot maintain these top speeds for long distances due to the intense energy expenditure required. They typically chase prey over short to medium distances before tiring and needing to rest. \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/object1.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/object1.0.result new file mode 100644 index 000000000..191028156 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/object1.0.result @@ -0,0 +1 @@ +foo \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/object2.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/object2.0.result new file mode 100644 index 000000000..05a514738 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/object2.0.result @@ -0,0 +1 @@ +foo2 \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/read-file.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/read-file.0.result new file mode 100644 index 000000000..8ca3e3fb2 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/read-file.0.result @@ -0,0 +1 @@ +{"a": {"b": 3}}3 \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/repeat1.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/repeat1.0.result new file mode 100644 index 000000000..df689d840 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/repeat1.0.result @@ -0,0 +1 @@ +234 \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/repeat2.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/repeat2.0.result new file mode 100644 index 000000000..3e195ef7d --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/repeat2.0.result @@ -0,0 +1 @@ +1a2b3c \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/repeat3.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/repeat3.0.result new file mode 100644 index 000000000..7ee61da82 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/repeat3.0.result @@ -0,0 +1 @@ +4a5b6c \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1.0.result new file mode 100644 index 000000000..ef2997800 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1.0.result @@ -0,0 +1 @@ +3yo3mo \ No newline at end of file diff --git a/tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1_wrapper.0.result b/tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1_wrapper.0.result new file mode 100644 index 000000000..ef2997800 --- /dev/null +++ b/tests/results/pdl-live-react/src-tauri/tests/cli/scoping_1_wrapper.0.result @@ -0,0 +1 @@ +3yo3mo \ No newline at end of file diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 76426ad28..81b66d229 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,4 +1,4 @@ -update_results: true +update_results: false check: [] skip: - examples/cldk/cldk-assistant.pdl From b7cda958ebdd4b14a2f32c9e168a10557f764a25 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 10:39:33 -0400 Subject: [PATCH 05/37] Small fix Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 6 ++---- tests/results/examples/demo/9-react.4.result | 14 ++++++++++++++ tests/results/examples/react/demo.5.result | 13 +++++++++++++ tests/test_examples_run.yaml | 5 ++++- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 tests/results/examples/demo/9-react.4.result create mode 100644 tests/results/examples/react/demo.5.result diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index e7197a1a3..ff2290a3c 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -50,10 +50,8 @@ jobs: with: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} - # update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} - # check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} - update-results: 'true' - check: '[]' + update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} + check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} head-ref: ${{ github.head_ref }} # Patch config with update_results to false diff --git a/tests/results/examples/demo/9-react.4.result b/tests/results/examples/demo/9-react.4.result new file mode 100644 index 000000000..679bcf2cd --- /dev/null +++ b/tests/results/examples/demo/9-react.4.result @@ -0,0 +1,14 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. He was born around 1565. +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Henry Hudson was born around 1565, so he was approximately 460 years ago.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: diff --git a/tests/results/examples/react/demo.5.result b/tests/results/examples/react/demo.5.result new file mode 100644 index 000000000..ce67247d3 --- /dev/null +++ b/tests/results/examples/react/demo.5.result @@ -0,0 +1,13 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson discovered the Hudson River in 1609. He was born around 1565. +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Henry Hudson was born around 1565, so he was approximately 460 years ago.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 81b66d229..eb457ccf1 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,5 +1,8 @@ update_results: false -check: [] +check: + - examples/chatbot/chatbot.pdl + - examples/demo/9-react.pdl + - examples/react/demo.pdl skip: - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl From 8e6d9de4e3e284060125f128d30636aa95433888 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:19:28 -0400 Subject: [PATCH 06/37] Rm dependency on yaml actions --- .github/actions/run-examples/action.yml | 13 ++++--------- .github/workflows/run-examples-prep.yml | 12 +++--------- tests/test_examples_run.yaml | 5 +---- 3 files changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index e370c69d7..94aebf395 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -30,15 +30,10 @@ runs: ref: ${{ inputs.head-ref }} fetch-depth: 0 # Full history needed for fetch/rebase - name: Patch tests/test_examples_run.yaml check with modified files - uses: fjogeleit/yaml-update-action@main - with: - changes: | - { - "tests/test_examples_run.yaml": { - "check": ${{ inputs.check }} - } - } - commitChange: false + shell: bash + run: | + yq -i '.check = (${{ inputs.check }} | from_json )' tests/test_examples_run.yaml + - name: Print test Run Examples config shell: bash run: cat tests/test_examples_run.yaml diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index ff2290a3c..622002af3 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -63,15 +63,9 @@ jobs: fetch-depth: 0 - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} - uses: fjogeleit/yaml-update-action@main - with: - changes: | - { - "tests/test_examples_run.yaml": { - "update_results": false - } - } - commitChange: false + shell: bash + run: | + yq -i '.update_results = ('false')' tests/test_examples_run.yaml - name: Push the new run-examples yaml config if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} shell: bash diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index eb457ccf1..fb317288b 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,8 +1,5 @@ update_results: false -check: - - examples/chatbot/chatbot.pdl - - examples/demo/9-react.pdl - - examples/react/demo.pdl +check: ["s1", "s2"] skip: - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl From dc216c9ef885998a04e32cac52684d21a9d96114 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:21:42 -0400 Subject: [PATCH 07/37] Add repo to checkout --- .github/workflows/run-examples-prep.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 622002af3..9c23c054e 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -59,6 +59,7 @@ jobs: - uses: actions/checkout@v4 if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} with: + repository: ${{ github.repository }} ref: ${{ github.head_ref }} fetch-depth: 0 - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version From c76a2a1f1c61b29e8ce25c3e963ec37006981c07 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:22:33 -0400 Subject: [PATCH 08/37] Update checkout ref --- .github/workflows/run-examples-prep.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 9c23c054e..2438c08be 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -59,8 +59,6 @@ jobs: - uses: actions/checkout@v4 if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} with: - repository: ${{ github.repository }} - ref: ${{ github.head_ref }} fetch-depth: 0 - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} From f4bd8fab31979d5087f22da3394458d7f7d32e39 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:26:28 -0400 Subject: [PATCH 09/37] Dont check out again --- .github/workflows/run-examples-prep.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 2438c08be..f3fa3ec38 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -55,11 +55,11 @@ jobs: head-ref: ${{ github.head_ref }} # Patch config with update_results to false - # Fetch the latest branch - - uses: actions/checkout@v4 - if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} - with: - fetch-depth: 0 + # # Fetch the latest branch + # - uses: actions/checkout@v4 + # if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} + # with: + # fetch-depth: 0 - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} shell: bash From 7a6a69eca4b89ddda1f2028f4eae6a102b59c734 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:27:23 -0400 Subject: [PATCH 10/37] Rm another checkout --- .github/workflows/run-examples-prep.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index f3fa3ec38..7b8f0c667 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -31,11 +31,11 @@ jobs: run: echo "$MODIFIED_PDL_FILES" # Run tests if there are modified PDL files - - uses: actions/checkout@v4 - if: ${{ steps.changed-pdl-files.outputs.all_changed_files_count > 0 }} - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 + # - uses: actions/checkout@v4 + # if: ${{ steps.changed-pdl-files.outputs.all_changed_files_count > 0 }} + # with: + # ref: ${{ github.head_ref }} + # fetch-depth: 0 - name: Check if update results automatically by GitHub Actions Bot id: parse-run-examples-config From 1bce4741f827b3e7540bb1758bee62c7112a823b Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:30:44 -0400 Subject: [PATCH 11/37] Rm checkout refs --- .github/actions/run-examples/action.yml | 4 +--- .github/workflows/run-examples-prep.yml | 12 +----------- tests/test_examples_run.yaml | 4 ++-- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index 94aebf395..47b4cf10c 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -26,9 +26,7 @@ runs: # Configure Run Examples environment - uses: actions/checkout@v4 - with: - ref: ${{ inputs.head-ref }} - fetch-depth: 0 # Full history needed for fetch/rebase + - name: Patch tests/test_examples_run.yaml check with modified files shell: bash run: | diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 7b8f0c667..0516b9e2f 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -12,6 +12,7 @@ jobs: max-parallel: 1 matrix: python-version: ['3.11', '3.12', '3.13'] + steps: # Detect modified PDL files, includes Add, Modified, but not Deleted - uses: actions/checkout@v4 @@ -31,12 +32,6 @@ jobs: run: echo "$MODIFIED_PDL_FILES" # Run tests if there are modified PDL files - # - uses: actions/checkout@v4 - # if: ${{ steps.changed-pdl-files.outputs.all_changed_files_count > 0 }} - # with: - # ref: ${{ github.head_ref }} - # fetch-depth: 0 - - name: Check if update results automatically by GitHub Actions Bot id: parse-run-examples-config shell: bash @@ -55,11 +50,6 @@ jobs: head-ref: ${{ github.head_ref }} # Patch config with update_results to false - # # Fetch the latest branch - # - uses: actions/checkout@v4 - # if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} - # with: - # fetch-depth: 0 - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} shell: bash diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index fb317288b..76426ad28 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,5 +1,5 @@ -update_results: false -check: ["s1", "s2"] +update_results: true +check: [] skip: - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl From 63830c4e4de1bdb97eed0673575510b27212b735 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:34:45 -0400 Subject: [PATCH 12/37] Fix yq --- .github/actions/run-examples/action.yml | 2 +- tests/test_examples_run.yaml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index 47b4cf10c..398252818 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -30,7 +30,7 @@ runs: - name: Patch tests/test_examples_run.yaml check with modified files shell: bash run: | - yq -i '.check = (${{ inputs.check }} | from_json )' tests/test_examples_run.yaml + yq -i '.check = (${{ inputs.check }})' tests/test_examples_run.yaml - name: Print test Run Examples config shell: bash diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 76426ad28..cdb81544e 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,5 +1,8 @@ update_results: true -check: [] +check: + - examples/chatbot/chatbot.pdl + - examples/demo/9-react.pdl + - examples/react/demo.pdl skip: - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl From 20445b5a254b7616b3ca56c7b732e13065b89059 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 18:56:29 -0400 Subject: [PATCH 13/37] Configure git and try again --- .github/actions/run-examples/action.yml | 16 +++++++++++++--- .github/workflows/run-examples-prep.yml | 5 ++++- .github/workflows/run-examples.yml | 3 +++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index 398252818..da4346ec4 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -8,9 +8,16 @@ inputs: runner-os: description: 'Runner OS' required: true + repository: + description: 'Repository name this pull request is initiated from' + required: false head-ref: description: 'Head ref of the repo' - required: true + required: false + default: 'main' + token: + description: 'Github token' + required: false update-results: description: 'Whether to update the results for this run. Must be false for nightly runs' required: true @@ -26,6 +33,9 @@ runs: # Configure Run Examples environment - uses: actions/checkout@v4 + with: + token: ${{ inputs.token }} + fetch-depth: 0 - name: Patch tests/test_examples_run.yaml check with modified files shell: bash @@ -81,10 +91,10 @@ runs: git config --local user.name github-actions[bot] git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com git status - git pull + git pull origin ${{ inputs.head-ref }} git add tests/results/ git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" - git push + git push origin ${{ inputs.head-ref }} - name: Check if pytest passed shell: bash diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 0516b9e2f..a0ff7abd3 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -17,6 +17,7 @@ jobs: # Detect modified PDL files, includes Add, Modified, but not Deleted - uses: actions/checkout@v4 with: + token: ${{ github.token }} fetch-depth: 0 - name: Detect all PDL files that were changed or added id: changed-pdl-files @@ -47,7 +48,9 @@ jobs: runner-os: ${{ runner.os }} update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} - head-ref: ${{ github.head_ref }} + head-ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.repository.full_name }} + token: ${{ github.token }} # Patch config with update_results to false - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version diff --git a/.github/workflows/run-examples.yml b/.github/workflows/run-examples.yml index 6cc92c3d3..abc7fd933 100644 --- a/.github/workflows/run-examples.yml +++ b/.github/workflows/run-examples.yml @@ -22,6 +22,9 @@ jobs: with: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} + repository: ${{ github.repository }} head-ref: ${{ github.head_ref }} + token: ${{ github.token }} update-results: 'false' check: '[]' # Empty list means run against all PDL programs + From cfcae08f0eb484715edbea78cd48ded82c41445b Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 19:14:09 -0400 Subject: [PATCH 14/37] Fix source repo name --- .github/workflows/run-examples-prep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index a0ff7abd3..b3ee5774e 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -49,7 +49,7 @@ jobs: update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} head-ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.repository.full_name }} + repository: ${{ github.event.pull_request.head.repo.full_name }} token: ${{ github.token }} # Patch config with update_results to false From dc314fcfdf8367c3f7b8cb51ca1992ebba2a2809 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 19:35:45 -0400 Subject: [PATCH 15/37] Checkout fix --- .github/actions/run-examples/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index da4346ec4..caf5d1435 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -35,6 +35,8 @@ runs: - uses: actions/checkout@v4 with: token: ${{ inputs.token }} + head-ref: ${{ inputs.head-ref }} + repository: ${{ inputs.repository }} fetch-depth: 0 - name: Patch tests/test_examples_run.yaml check with modified files From 559b926e7146bd8f6613cf55b59cb26c789c2f2e Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 19:41:01 -0400 Subject: [PATCH 16/37] Fix ref --- .github/actions/run-examples/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index caf5d1435..a1776b24b 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -35,7 +35,7 @@ runs: - uses: actions/checkout@v4 with: token: ${{ inputs.token }} - head-ref: ${{ inputs.head-ref }} + ref: ${{ inputs.head-ref }} repository: ${{ inputs.repository }} fetch-depth: 0 From 79da4812d2c00105e44ec9d02fb2817d6918366b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 00:08:32 +0000 Subject: [PATCH 17/37] github-actions[bot]: Run examples: reverting update_result to false on your behalf Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/test_examples_run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index cdb81544e..6059ed275 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,4 +1,4 @@ -update_results: true +update_results: false check: - examples/chatbot/chatbot.pdl - examples/demo/9-react.pdl From 2457314f459129ec4ee9fc7004cf22711bec3800 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 20:11:59 -0400 Subject: [PATCH 18/37] Try push again --- .github/actions/run-examples/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-examples/action.yml b/.github/actions/run-examples/action.yml index a1776b24b..9dc8205bb 100644 --- a/.github/actions/run-examples/action.yml +++ b/.github/actions/run-examples/action.yml @@ -96,7 +96,8 @@ runs: git pull origin ${{ inputs.head-ref }} git add tests/results/ git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" - git push origin ${{ inputs.head-ref }} + # git push origin ${{ inputs.head-ref }} + git push https://x-access-token:${{ inputs.token }}@github.com/${{ inputs.repository }} HEAD:${{ inputs.head-ref }} - name: Check if pytest passed shell: bash From 7cd24ca1b3c78deb23d9c2121ed88f1f060296c1 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 20:12:41 -0400 Subject: [PATCH 19/37] Fix update results bool --- tests/test_examples_run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 6059ed275..cdb81544e 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,4 +1,4 @@ -update_results: false +update_results: true check: - examples/chatbot/chatbot.pdl - examples/demo/9-react.pdl From e21bbd130db743f15b9b3456acbfc68bd9e1deca Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 20:25:17 -0400 Subject: [PATCH 20/37] Try another push --- .github/workflows/run-examples-prep.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index b3ee5774e..0b29a3655 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -17,6 +17,8 @@ jobs: # Detect modified PDL files, includes Add, Modified, but not Deleted - uses: actions/checkout@v4 with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} token: ${{ github.token }} fetch-depth: 0 - name: Detect all PDL files that were changed or added From f5232cc5f35f471c332c87361dd35dbd819f926e Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Sun, 13 Apr 2025 20:35:54 -0400 Subject: [PATCH 21/37] Try using secrets --- .github/workflows/run-examples-prep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 0b29a3655..f2a9966ec 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -52,7 +52,7 @@ jobs: check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} head-ref: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ github.token }} + token: ${{ secrets.PDL_PAT }} # Patch config with update_results to false - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version From d601797c9e4323ad7e12d06d52bf19e1f4129a5e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 00:40:52 +0000 Subject: [PATCH 22/37] github-actions[bot]: Run examples: reverting update_result to false on your behalf Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tests/test_examples_run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index cdb81544e..6059ed275 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,4 +1,4 @@ -update_results: true +update_results: false check: - examples/chatbot/chatbot.pdl - examples/demo/9-react.pdl From af5e740f821f25723718a22e349b2f7184d5d599 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Mon, 14 Apr 2025 19:39:00 -0400 Subject: [PATCH 23/37] Don't use composite action in pr --- .github/workflows/run-examples-prep.yml | 115 +++++++++++++++++++----- 1 file changed, 92 insertions(+), 23 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index f2a9966ec..ab5e24c86 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -17,9 +17,6 @@ jobs: # Detect modified PDL files, includes Add, Modified, but not Deleted - uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ github.token }} fetch-depth: 0 - name: Detect all PDL files that were changed or added id: changed-pdl-files @@ -30,38 +27,104 @@ jobs: json: "true" escape_json: "false" - name: List PDL files that were modified or added and append to test_examples_run - env: - MODIFIED_PDL_FILES: ${{ steps.changed-pdl-files.outputs.all_changed_files }} - run: echo "$MODIFIED_PDL_FILES" + run: | + echo ${{ steps.changed-pdl-files.outputs.all_changed_files }} + if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then + echo "early-stop=false" >> $GITHUB_ENV + else + echo "No file need to be checked, skipping all subsequent tests." + echo "early-stop=true" >> $GITHUB_ENV + fi # Run tests if there are modified PDL files - name: Check if update results automatically by GitHub Actions Bot + if: ${{ !env.early-stop }} id: parse-run-examples-config shell: bash run: | value=$(yq '.update_results' tests/test_examples_run.yaml | tr '[:upper:]' '[:lower:]') echo "value=$value" - echo "update_results=$value" >> $GITHUB_OUTPUT + echo "update_results=$value" >> $GITHUB_ENV + + # Set up Ollama + - name: Cache Ollama model files + if: ${{ !env.early-stop }} + uses: actions/cache@v4 + with: + path: /usr/share/ollama/.ollama/models + key: ${{ runner.os }}-build-ollama-cache + restore-keys: | + ${{ runner.os }}-build- + ${{ runner.os }} + - uses: ./.github/actions/ollama + if: ${{ !env.early-stop }} - - uses: ./.github/actions/run-examples - if: ${{ steps.changed-pdl-files.outputs.all_changed_files_count > 0 }} + # Set up Run Examples environment + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + if: ${{ !env.early-stop }} with: python-version: ${{ matrix.python-version }} - runner-os: ${{ runner.os }} - update-results: ${{ steps.parse-run-examples-config.outputs.update_results }} - check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} - head-ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.PDL_PAT }} + - name: Cache pip + uses: actions/cache@v4 + if: ${{ !env.early-stop }} + with: + # This path is specific to Ubuntu + path: ${{ env.pythonLocation }} + # Look to see if there is a cache hit for the setup file + key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} + restore-keys: | + ${{ runner.os }}-pip-new3 + ${{ runner.os }}-new3 + - name: Install dependencies + if: ${{ !env.early-stop }} + shell: bash + run: pip install --upgrade --upgrade-strategy eager .[all] + - name: Pip list packages + if: ${{ !env.early-stop }} + shell: bash + run: pip list + - name: View Run Examples config + if: ${{ !env.early-stop }} + shell: bash + run: cat tests/test_examples_run.yaml + - name: Run Pytest + if: ${{ !env.early-stop }} + shell: bash + run: | + ( + set +e + py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings + EXIT_CODE=$? - # Patch config with update_results to false - - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last matrix version - if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} + if [ $EXIT_CODE -eq 0 ]; then + echo "TEST_RESULT=PASSED" >> $GITHUB_ENV + else + echo "TEST_RESULT=FAILED" >> $GITHUB_ENV + fi + ) + + # Commit the results if update results + - name: Push new results to branch + shell: bash + if: ${{ !env.early-stop && env.update_results }} + run: | + git config --local user.name github-actions[bot] + git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com + git status + git pull + git add tests/results/ + git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" + git push + + # Patch config with update_results to false and commit to PR + - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last job in the series + if: ${{ !env.early-stop && env.update_results && matrix.python-version == '3.13' }} shell: bash run: | yq -i '.update_results = ('false')' tests/test_examples_run.yaml - name: Push the new run-examples yaml config - if: ${{ steps.parse-run-examples-config.outputs.update_results && matrix.python-version == '3.13' }} + if: ${{ !env.early-stop && env.update_results && matrix.python-version == '3.13' }} shell: bash run: | git config --local user.name github-actions[bot] @@ -71,8 +134,14 @@ jobs: git add tests/test_examples_run.yaml git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: reverting update_result to false on your behalf" git push - - - - + + - name: Check if pytest passed + shell: bash + run: | + if [ "$TEST_RESULT" == "PASSED" ]; then + exit 0 + else + exit 1 + fi + \ No newline at end of file From 3905db1f78746657b2852fce728cb71cc1676666 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Mon, 14 Apr 2025 19:43:00 -0400 Subject: [PATCH 24/37] Refine bool comparison --- .github/workflows/run-examples-prep.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index ab5e24c86..c15885d92 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -38,7 +38,7 @@ jobs: # Run tests if there are modified PDL files - name: Check if update results automatically by GitHub Actions Bot - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} id: parse-run-examples-config shell: bash run: | @@ -48,7 +48,7 @@ jobs: # Set up Ollama - name: Cache Ollama model files - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} uses: actions/cache@v4 with: path: /usr/share/ollama/.ollama/models @@ -57,17 +57,17 @@ jobs: ${{ runner.os }}-build- ${{ runner.os }} - uses: ./.github/actions/ollama - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} # Set up Run Examples environment - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} with: python-version: ${{ matrix.python-version }} - name: Cache pip uses: actions/cache@v4 - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} with: # This path is specific to Ubuntu path: ${{ env.pythonLocation }} @@ -77,19 +77,19 @@ jobs: ${{ runner.os }}-pip-new3 ${{ runner.os }}-new3 - name: Install dependencies - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} shell: bash run: pip install --upgrade --upgrade-strategy eager .[all] - name: Pip list packages - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} shell: bash run: pip list - name: View Run Examples config - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} shell: bash run: cat tests/test_examples_run.yaml - name: Run Pytest - if: ${{ !env.early-stop }} + if: ${{ env.early-stop == 'false' }} shell: bash run: | ( @@ -107,7 +107,7 @@ jobs: # Commit the results if update results - name: Push new results to branch shell: bash - if: ${{ !env.early-stop && env.update_results }} + if: ${{ env.early-stop == 'false' && env.update_results == 'true' }} run: | git config --local user.name github-actions[bot] git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com @@ -119,12 +119,12 @@ jobs: # Patch config with update_results to false and commit to PR - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last job in the series - if: ${{ !env.early-stop && env.update_results && matrix.python-version == '3.13' }} + if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} shell: bash run: | yq -i '.update_results = ('false')' tests/test_examples_run.yaml - name: Push the new run-examples yaml config - if: ${{ !env.early-stop && env.update_results && matrix.python-version == '3.13' }} + if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} shell: bash run: | git config --local user.name github-actions[bot] @@ -137,6 +137,7 @@ jobs: - name: Check if pytest passed shell: bash + if: ${{ env.early-stop == 'false' }} run: | if [ "$TEST_RESULT" == "PASSED" ]; then exit 0 From fec39b18be8e5fbf33183b62c6d6d2faaa3dce2d Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Mon, 14 Apr 2025 20:23:59 -0400 Subject: [PATCH 25/37] Don't cache ollama --- .github/workflows/run-examples-prep.yml | 18 +++++++++--------- tests/test_examples_run.yaml | 7 ++----- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index c15885d92..c112424df 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -47,15 +47,15 @@ jobs: echo "update_results=$value" >> $GITHUB_ENV # Set up Ollama - - name: Cache Ollama model files - if: ${{ env.early-stop == 'false' }} - uses: actions/cache@v4 - with: - path: /usr/share/ollama/.ollama/models - key: ${{ runner.os }}-build-ollama-cache - restore-keys: | - ${{ runner.os }}-build- - ${{ runner.os }} + # - name: Cache Ollama model files + # if: ${{ env.early-stop == 'false' }} + # uses: actions/cache@v4 + # with: + # path: /usr/share/ollama/.ollama/models + # key: ${{ runner.os }}-build-ollama-cache + # restore-keys: | + # ${{ runner.os }}-build- + # ${{ runner.os }} - uses: ./.github/actions/ollama if: ${{ env.early-stop == 'false' }} diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 6059ed275..76426ad28 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,8 +1,5 @@ -update_results: false -check: - - examples/chatbot/chatbot.pdl - - examples/demo/9-react.pdl - - examples/react/demo.pdl +update_results: true +check: [] skip: - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl From 85618bd02047b41f96015aad8e0d412dc607bbf4 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Mon, 14 Apr 2025 20:51:28 -0400 Subject: [PATCH 26/37] Add patch yaml step --- .github/workflows/run-examples-prep.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index c112424df..7e54d3046 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -84,6 +84,11 @@ jobs: if: ${{ env.early-stop == 'false' }} shell: bash run: pip list + - name: Patch tests/test_examples_run.yaml check with modified files + if: ${{ env.early-stop == 'false' }} + shell: bash + run: | + yq -i '.check = (${{ steps.changed-pdl-files.outputs.all_changed_files }})' tests/test_examples_run.yaml - name: View Run Examples config if: ${{ env.early-stop == 'false' }} shell: bash From 6db7ba3cf40b28654d0bcb417269180e98a7934d Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Fri, 18 Apr 2025 10:17:50 -0400 Subject: [PATCH 27/37] Test if PAT works --- .github/workflows/run-examples-prep.yml | 273 +++++++++++++----------- 1 file changed, 144 insertions(+), 129 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 7e54d3046..d44a1bcda 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -7,147 +7,162 @@ jobs: runs-on: ubuntu-latest permissions: contents: write - strategy: - fail-fast: false - max-parallel: 1 - matrix: - python-version: ['3.11', '3.12', '3.13'] + # strategy: + # fail-fast: false + # max-parallel: 1 + # matrix: + # python-version: ['3.11', '3.12', '3.13'] steps: - # Detect modified PDL files, includes Add, Modified, but not Deleted - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Detect all PDL files that were changed or added - id: changed-pdl-files - uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46 - with: - files: | - **.pdl - json: "true" - escape_json: "false" - - name: List PDL files that were modified or added and append to test_examples_run + + - name: Push the new run-examples yaml config + shell: bash run: | - echo ${{ steps.changed-pdl-files.outputs.all_changed_files }} - if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then - echo "early-stop=false" >> $GITHUB_ENV - else - echo "No file need to be checked, skipping all subsequent tests." - echo "early-stop=true" >> $GITHUB_ENV - fi + echo "This is a test: $(date)" > test-file.txt + git config --local user.name github-actions[bot] + git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com + git remote set-url fork-remote https://${{ secrets.PDL_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git + git status + git pull fork-remote ${{ github.head_ref }} + git add test-file.txt + git diff --cached --quiet || git commit -s -m "github-actions[bot]: Simple test for PAT" + git push fork-remote ${{ github.head_ref }} + + # # Detect modified PDL files, includes Add, Modified, but not Deleted + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # - name: Detect all PDL files that were changed or added + # id: changed-pdl-files + # uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46 + # with: + # files: | + # **.pdl + # json: "true" + # escape_json: "false" + # - name: List PDL files that were modified or added and append to test_examples_run + # run: | + # echo ${{ steps.changed-pdl-files.outputs.all_changed_files }} + # if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then + # echo "early-stop=false" >> $GITHUB_ENV + # else + # echo "No file need to be checked, skipping all subsequent tests." + # echo "early-stop=true" >> $GITHUB_ENV + # fi - # Run tests if there are modified PDL files - - name: Check if update results automatically by GitHub Actions Bot - if: ${{ env.early-stop == 'false' }} - id: parse-run-examples-config - shell: bash - run: | - value=$(yq '.update_results' tests/test_examples_run.yaml | tr '[:upper:]' '[:lower:]') - echo "value=$value" - echo "update_results=$value" >> $GITHUB_ENV + # # Run tests if there are modified PDL files + # - name: Check if update results automatically by GitHub Actions Bot + # if: ${{ env.early-stop == 'false' }} + # id: parse-run-examples-config + # shell: bash + # run: | + # value=$(yq '.update_results' tests/test_examples_run.yaml | tr '[:upper:]' '[:lower:]') + # echo "value=$value" + # echo "update_results=$value" >> $GITHUB_ENV - # Set up Ollama - # - name: Cache Ollama model files + # # Set up Ollama + # # - name: Cache Ollama model files + # # if: ${{ env.early-stop == 'false' }} + # # uses: actions/cache@v4 + # # with: + # # path: /usr/share/ollama/.ollama/models + # # key: ${{ runner.os }}-build-ollama-cache + # # restore-keys: | + # # ${{ runner.os }}-build- + # # ${{ runner.os }} + # - uses: ./.github/actions/ollama + # if: ${{ env.early-stop == 'false' }} + + # # Set up Run Examples environment + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 # if: ${{ env.early-stop == 'false' }} + # with: + # python-version: ${{ matrix.python-version }} + # - name: Cache pip # uses: actions/cache@v4 + # if: ${{ env.early-stop == 'false' }} # with: - # path: /usr/share/ollama/.ollama/models - # key: ${{ runner.os }}-build-ollama-cache + # # This path is specific to Ubuntu + # path: ${{ env.pythonLocation }} + # # Look to see if there is a cache hit for the setup file + # key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} # restore-keys: | - # ${{ runner.os }}-build- - # ${{ runner.os }} - - uses: ./.github/actions/ollama - if: ${{ env.early-stop == 'false' }} - - # Set up Run Examples environment - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - if: ${{ env.early-stop == 'false' }} - with: - python-version: ${{ matrix.python-version }} - - name: Cache pip - uses: actions/cache@v4 - if: ${{ env.early-stop == 'false' }} - with: - # This path is specific to Ubuntu - path: ${{ env.pythonLocation }} - # Look to see if there is a cache hit for the setup file - key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - restore-keys: | - ${{ runner.os }}-pip-new3 - ${{ runner.os }}-new3 - - name: Install dependencies - if: ${{ env.early-stop == 'false' }} - shell: bash - run: pip install --upgrade --upgrade-strategy eager .[all] - - name: Pip list packages - if: ${{ env.early-stop == 'false' }} - shell: bash - run: pip list - - name: Patch tests/test_examples_run.yaml check with modified files - if: ${{ env.early-stop == 'false' }} - shell: bash - run: | - yq -i '.check = (${{ steps.changed-pdl-files.outputs.all_changed_files }})' tests/test_examples_run.yaml - - name: View Run Examples config - if: ${{ env.early-stop == 'false' }} - shell: bash - run: cat tests/test_examples_run.yaml - - name: Run Pytest - if: ${{ env.early-stop == 'false' }} - shell: bash - run: | - ( - set +e - py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings - EXIT_CODE=$? + # ${{ runner.os }}-pip-new3 + # ${{ runner.os }}-new3 + # - name: Install dependencies + # if: ${{ env.early-stop == 'false' }} + # shell: bash + # run: pip install --upgrade --upgrade-strategy eager .[all] + # - name: Pip list packages + # if: ${{ env.early-stop == 'false' }} + # shell: bash + # run: pip list + # - name: Patch tests/test_examples_run.yaml check with modified files + # if: ${{ env.early-stop == 'false' }} + # shell: bash + # run: | + # yq -i '.check = (${{ steps.changed-pdl-files.outputs.all_changed_files }})' tests/test_examples_run.yaml + # - name: View Run Examples config + # if: ${{ env.early-stop == 'false' }} + # shell: bash + # run: cat tests/test_examples_run.yaml + # - name: Run Pytest + # if: ${{ env.early-stop == 'false' }} + # shell: bash + # run: | + # ( + # set +e + # py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings + # EXIT_CODE=$? - if [ $EXIT_CODE -eq 0 ]; then - echo "TEST_RESULT=PASSED" >> $GITHUB_ENV - else - echo "TEST_RESULT=FAILED" >> $GITHUB_ENV - fi - ) + # if [ $EXIT_CODE -eq 0 ]; then + # echo "TEST_RESULT=PASSED" >> $GITHUB_ENV + # else + # echo "TEST_RESULT=FAILED" >> $GITHUB_ENV + # fi + # ) - # Commit the results if update results - - name: Push new results to branch - shell: bash - if: ${{ env.early-stop == 'false' && env.update_results == 'true' }} - run: | - git config --local user.name github-actions[bot] - git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com - git status - git pull - git add tests/results/ - git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" - git push + # # Commit the results if update results + # - name: Push new results to branch + # shell: bash + # if: ${{ env.early-stop == 'false' && env.update_results == 'true' }} + # run: | + # git config --local user.name github-actions[bot] + # git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com + # git status + # git pull + # git add tests/results/ + # git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" + # git push - # Patch config with update_results to false and commit to PR - - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last job in the series - if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} - shell: bash - run: | - yq -i '.update_results = ('false')' tests/test_examples_run.yaml - - name: Push the new run-examples yaml config - if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} - shell: bash - run: | - git config --local user.name github-actions[bot] - git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com - git status - git pull - git add tests/test_examples_run.yaml - git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: reverting update_result to false on your behalf" - git push + # # Patch config with update_results to false and commit to PR + # - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last job in the series + # if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} + # shell: bash + # run: | + # yq -i '.update_results = ('false')' tests/test_examples_run.yaml + # - name: Push the new run-examples yaml config + # if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} + # shell: bash + # run: | + # git config --local user.name github-actions[bot] + # git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com + # git remote set-url origin https://${{ secrets.PDL_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git + # git status + # git pull origin ${{ github.head_ref }} + # git add tests/test_examples_run.yaml + # git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: reverting update_result to false on your behalf" + # git push origin ${{ github.head_ref }} - - name: Check if pytest passed - shell: bash - if: ${{ env.early-stop == 'false' }} - run: | - if [ "$TEST_RESULT" == "PASSED" ]; then - exit 0 - else - exit 1 - fi + # - name: Check if pytest passed + # shell: bash + # if: ${{ env.early-stop == 'false' }} + # run: | + # if [ "$TEST_RESULT" == "PASSED" ]; then + # exit 0 + # else + # exit 1 + # fi \ No newline at end of file From 7c906385ef19f2244bd6dba82724f09b9f378553 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Fri, 18 Apr 2025 10:23:06 -0400 Subject: [PATCH 28/37] Add checkout step --- .github/workflows/run-examples-prep.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index d44a1bcda..f322fb0f3 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -14,6 +14,10 @@ jobs: # python-version: ['3.11', '3.12', '3.13'] steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} - name: Push the new run-examples yaml config shell: bash From 42133d6ed7158ab7eec00ef716f18792749ccdfa Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Fri, 18 Apr 2025 10:24:26 -0400 Subject: [PATCH 29/37] Checkout modification --- .github/workflows/run-examples-prep.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index f322fb0f3..72a28dc64 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -16,8 +16,10 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.PDL_PAT }} + fetch-depth: 0 - name: Push the new run-examples yaml config shell: bash From 3dc0ca12149028dbb510a15f89faae7ae370d5d2 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 11:57:22 -0400 Subject: [PATCH 30/37] Update with feedback Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 199 +++++------------------- docs/contrib.md | 21 +-- tests/test_examples_run.yaml | 18 ++- 3 files changed, 61 insertions(+), 177 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 72a28dc64..f63241477 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -5,170 +5,49 @@ jobs: tests: name: Execution tests runs-on: ubuntu-latest - permissions: - contents: write - # strategy: - # fail-fast: false - # max-parallel: 1 - # matrix: - # python-version: ['3.11', '3.12', '3.13'] + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python-version: ['3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - token: ${{ secrets.PDL_PAT }} + ref: ${{ github.head_ref }} fetch-depth: 0 - - - name: Push the new run-examples yaml config - shell: bash - run: | - echo "This is a test: $(date)" > test-file.txt - git config --local user.name github-actions[bot] - git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com - git remote set-url fork-remote https://${{ secrets.PDL_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git - git status - git pull fork-remote ${{ github.head_ref }} - git add test-file.txt - git diff --cached --quiet || git commit -s -m "github-actions[bot]: Simple test for PAT" - git push fork-remote ${{ github.head_ref }} - # # Detect modified PDL files, includes Add, Modified, but not Deleted - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # - name: Detect all PDL files that were changed or added - # id: changed-pdl-files - # uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46 - # with: - # files: | - # **.pdl - # json: "true" - # escape_json: "false" - # - name: List PDL files that were modified or added and append to test_examples_run - # run: | - # echo ${{ steps.changed-pdl-files.outputs.all_changed_files }} - # if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then - # echo "early-stop=false" >> $GITHUB_ENV - # else - # echo "No file need to be checked, skipping all subsequent tests." - # echo "early-stop=true" >> $GITHUB_ENV - # fi - - # # Run tests if there are modified PDL files - # - name: Check if update results automatically by GitHub Actions Bot - # if: ${{ env.early-stop == 'false' }} - # id: parse-run-examples-config - # shell: bash - # run: | - # value=$(yq '.update_results' tests/test_examples_run.yaml | tr '[:upper:]' '[:lower:]') - # echo "value=$value" - # echo "update_results=$value" >> $GITHUB_ENV - - # # Set up Ollama - # # - name: Cache Ollama model files - # # if: ${{ env.early-stop == 'false' }} - # # uses: actions/cache@v4 - # # with: - # # path: /usr/share/ollama/.ollama/models - # # key: ${{ runner.os }}-build-ollama-cache - # # restore-keys: | - # # ${{ runner.os }}-build- - # # ${{ runner.os }} - # - uses: ./.github/actions/ollama - # if: ${{ env.early-stop == 'false' }} - - # # Set up Run Examples environment - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v5 - # if: ${{ env.early-stop == 'false' }} - # with: - # python-version: ${{ matrix.python-version }} - # - name: Cache pip - # uses: actions/cache@v4 - # if: ${{ env.early-stop == 'false' }} - # with: - # # This path is specific to Ubuntu - # path: ${{ env.pythonLocation }} - # # Look to see if there is a cache hit for the setup file - # key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }} - # restore-keys: | - # ${{ runner.os }}-pip-new3 - # ${{ runner.os }}-new3 - # - name: Install dependencies - # if: ${{ env.early-stop == 'false' }} - # shell: bash - # run: pip install --upgrade --upgrade-strategy eager .[all] - # - name: Pip list packages - # if: ${{ env.early-stop == 'false' }} - # shell: bash - # run: pip list - # - name: Patch tests/test_examples_run.yaml check with modified files - # if: ${{ env.early-stop == 'false' }} - # shell: bash - # run: | - # yq -i '.check = (${{ steps.changed-pdl-files.outputs.all_changed_files }})' tests/test_examples_run.yaml - # - name: View Run Examples config - # if: ${{ env.early-stop == 'false' }} - # shell: bash - # run: cat tests/test_examples_run.yaml - # - name: Run Pytest - # if: ${{ env.early-stop == 'false' }} - # shell: bash - # run: | - # ( - # set +e - # py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-warnings - # EXIT_CODE=$? - - # if [ $EXIT_CODE -eq 0 ]; then - # echo "TEST_RESULT=PASSED" >> $GITHUB_ENV - # else - # echo "TEST_RESULT=FAILED" >> $GITHUB_ENV - # fi - # ) - - # # Commit the results if update results - # - name: Push new results to branch - # shell: bash - # if: ${{ env.early-stop == 'false' && env.update_results == 'true' }} - # run: | - # git config --local user.name github-actions[bot] - # git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com - # git status - # git pull - # git add tests/results/ - # git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: updated result files on your behalf" - # git push - - # # Patch config with update_results to false and commit to PR - # - name: Patch tests/test_examples_run.yaml with update_results to False, only for the last job in the series - # if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} - # shell: bash - # run: | - # yq -i '.update_results = ('false')' tests/test_examples_run.yaml - # - name: Push the new run-examples yaml config - # if: ${{ env.early-stop == 'false' && env.update_results == 'true' && matrix.python-version == '3.13' }} - # shell: bash - # run: | - # git config --local user.name github-actions[bot] - # git config --local user.email 41898282+github-actions[bot]@users.noreply.github.com - # git remote set-url origin https://${{ secrets.PDL_PAT }}@github.com/${{ github.event.pull_request.head.repo.full_name }}.git - # git status - # git pull origin ${{ github.head_ref }} - # git add tests/test_examples_run.yaml - # git diff --cached --quiet || git commit -s -m "github-actions[bot]: Run examples: reverting update_result to false on your behalf" - # git push origin ${{ github.head_ref }} - - # - name: Check if pytest passed - # shell: bash - # if: ${{ env.early-stop == 'false' }} - # run: | - # if [ "$TEST_RESULT" == "PASSED" ]; then - # exit 0 - # else - # exit 1 - # fi + # Detect modified PDL files, includes Add, Modified, but not Deleted + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Detect all PDL files that were changed or added + id: changed-pdl-files + uses: tj-actions/changed-files@6cb76d07bee4c9772c6882c06c37837bf82a04d3 # v46 + with: + files: | + **.pdl + json: "true" + escape_json: "false" + - name: List PDL files that were modified or added and append to test_examples_run.yaml + run: | + echo ${{ steps.changed-pdl-files.outputs.all_changed_files }} + if [ ${{ steps.changed-pdl-files.outputs.all_changed_files_count }} -gt 0 ]; then + echo "early-stop=false" >> $GITHUB_ENV + else + echo "No file need to be checked, skipping all subsequent tests." + echo "early-stop=true" >> $GITHUB_ENV + fi - \ No newline at end of file + - name: Run examples + uses: ./.github/actions/run-examples + # Only run if there are modified PDL files + if: env.early-stop == 'false' + with: + python-version: ${{ matrix.python-version }} + runner-os: ${{ runner.os }} + repository: ${{ github.repository }} + head-ref: ${{ github.head_ref }} + token: ${{ github.token }} + update-results: 'false' + check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} \ No newline at end of file diff --git a/docs/contrib.md b/docs/contrib.md index 9fb51cc39..7f66d5174 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -48,7 +48,7 @@ Follow the following instructions to set up a dev environment to get started wit 4. Test that you can run an editable version of PDL ``` - pdl examples/hello/hello.pdl + pdl examples/demo/1-hello.pdl Hello Hello! How can I help you today? @@ -74,12 +74,12 @@ You are all set! ### Run examples -PDL executes nightly runs for Run Examples, which searches for all the `.pdl` programs in the repo and runs the interpreter against each file. The [config file for Run Examples](../tests/test_examples_run.yaml) describes how to handle with each file. There are four conditions: +PDL executes nightly runs for Run Examples, which searches for all the `.pdl` programs in the repo and runs the interpreter against each file. The [config file for Run Examples](../tests/test_examples_run.yaml) describes how to handle each file. There are four conditions: 1. `skip`: a list of PDL files that are skipped in Run Examples 2. `with_inputs`: PDL files that require user input. Each file name is mapped to two fields that describe how inputs are patched - 1. `stdin`: separated by a newline, each line represents the user input - 2. `scope`: scope for the PDL program + 1. `stdin`: separated by a newline, each line represents the user input (string) + 2. `scope`: scope for the PDL program (a dictionary of key-value pairs) 3. `expected_parse_error`: a list of PDL files that expect parse errors 4. `expected_runtime_error`: a list of PDL files that expect runtime errors @@ -99,18 +99,11 @@ pytest --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-war #### Opening a pull request -A slight variation in the Python version and OS environment can cause a different LLM response, thus Run Examples might fail because it uses exact string matching for PDL outputs. You can utilize the `update_results` when opening a PR to capture the expected results in the GH Actions environment. +A slight variation in the Python version and OS environment can cause a different LLM response, thus Run Examples might fail because it uses exact string matching for PDL outputs. -When you open a pull request (PR) against the `main` branch, a series of status checks will be executed. Specificially, three Run Examples test will be initiated against the files you have added and modified. +When you open a pull request (PR) against the `main` branch, a series of status checks will be executed. Specificially, three Run Examples test will be initiated against the files you have added and modified. If there's any variation, you should manually examine the results produced in the Github Actions environment, then copy and paste the results to a new file, and push another commit to your PR so the CI can pass. Be aware of whitespaces in between sentences, as the CI does an exact string matching. -- If in the PR, `update_results: false`, - - If Pytest fails, you can manually examine the failures and add the results to your PR -- If in the PR, `update_results: true`, - - If Pytest fails, a GH-Actions bot will push a commit with the new results produced in the GH environment to your PR - - Additionally, another commit will push `update_results: false`. This is to ensure that for the nightly test, different results are not committed - - If you need to make other changes to your PR, make sure to pull the new changes first - -Your PR should always set `update_results: false` before merging. +Note: Your PR should always set `update_results: false` before merging, though the CI pipeline will not attempt to write the results to your PR even if you forget to set this field to `false`. Here's a preview of the current configuration file for Run Examples: diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 76426ad28..3b4c3f23b 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -4,10 +4,10 @@ skip: - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl - examples/gsm8k/gsm8k-plan.pdl - - examples/gsm8k/gsm8k-tot.pdl - - examples/gsm8k/gsm8k-tot-multiplan.pdl - - examples/gsm8k/gsm8k-tot-few-shot.pdl - examples/gsm8k/gsm8k-plan-few-shots.pdl + - examples/gsm8k/gsm8k-tot-few-shot.pdl + - examples/gsm8k/gsm8k-tot-multiplan.pdl + - examples/gsm8k/gsm8k-tot.pdl - examples/intrinsics/demo-hallucination.pdl - examples/tutorial/programs/demo-hallucination.pdl - examples/rag/pdf_index.pdl @@ -15,6 +15,7 @@ skip: - examples/rag/rag_library1.pdl - examples/tutorial/structured_decoding.pdl - examples/react/react_call.pdl + - pdl-live-react/src-tauri/tests/cli/code-python.pdl - pdl-live-react/demos/error.pdl - pdl-live-react/demos/demo1.pdl - pdl-live-react/demos/demo2.pdl @@ -23,6 +24,9 @@ skip: - examples/granite-io/granite_io_openai.pdl - examples/granite-io/granite_io_thinking.pdl - examples/granite-io/granite_io_transformers.pdl + - examples/optimizer/mbpp.pdl + - examples/optimizer/fever.pdl + - examples/optimizer/gsm8k.pdl with_inputs: examples/tutorial/programs/chatbot.pdl: stdin: | @@ -61,6 +65,14 @@ with_inputs: stdin: null scope: something: ABC + tests/data/optimizer_gsm8k.pdl: + scope: + model: watsonx_text/ibm/granite-3-8b-instruct + prompt_pattern: cot + num_demonstrations: 0 + demonstrations: [] + question: "The sky currently has 4 times as many cirrus clouds as cumulus clouds, and 12 times as many cumulus clouds as cumulonimbus clouds. If the sky currently has 3 cumulonimbus clouds, how many cirrus clouds are in the sky at this moment?" + expected_parse_error: - tests/data/line/hello.pdl - tests/data/line/hello1.pdl From 161901e5ab2c15815d2bdfeb77df1a86537c4b5e Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 13:37:44 -0400 Subject: [PATCH 31/37] Fix checkout Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index f63241477..1d77e70d3 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -15,6 +15,7 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.head_ref }} + repository: ${{ github.repository }} fetch-depth: 0 # Detect modified PDL files, includes Add, Modified, but not Deleted From 67f5497a7b84cc585249b7817e6866f500522d50 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 13:45:25 -0400 Subject: [PATCH 32/37] Checkout fix Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 1d77e70d3..341449894 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -12,11 +12,10 @@ jobs: python-version: ['3.11', '3.12', '3.13'] steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - repository: ${{ github.repository }} - fetch-depth: 0 + # - uses: actions/checkout@v4 + # with: + # repository: ${{ github.event.pull_request.head.repo.full_name }} + # ref: ${{ github.event.pull_request.head.ref }} # Detect modified PDL files, includes Add, Modified, but not Deleted - uses: actions/checkout@v4 From a2b25bdc23a9669c7c5f41f9d339830fbe460131 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 13:49:40 -0400 Subject: [PATCH 33/37] Checkout fix 2 Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index 341449894..df7f86089 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -47,7 +47,7 @@ jobs: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} repository: ${{ github.repository }} - head-ref: ${{ github.head_ref }} + head-ref: ${{ github.event.pull_request.head.ref }} token: ${{ github.token }} update-results: 'false' check: ${{ steps.changed-pdl-files.outputs.all_changed_files }} \ No newline at end of file From a2949e344785e7e45d089ce35819ba0b777ddc9c Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 13:54:25 -0400 Subject: [PATCH 34/37] Checkout fix 3 Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index df7f86089..bc853ca8a 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -46,7 +46,7 @@ jobs: with: python-version: ${{ matrix.python-version }} runner-os: ${{ runner.os }} - repository: ${{ github.repository }} + repository: ${{ github.event.pull_request.head.repo.full_name }} head-ref: ${{ github.event.pull_request.head.ref }} token: ${{ github.token }} update-results: 'false' From c17a186375a5202e06ae7438ae5d62f02911084f Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 14:36:23 -0400 Subject: [PATCH 35/37] Update code Signed-off-by: Jing Chen --- docs/contrib.md | 2 +- tests/test_examples_run.py | 11 +++++++---- tests/test_examples_run.yaml | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/contrib.md b/docs/contrib.md index 7f66d5174..fd4b51387 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -51,7 +51,7 @@ Follow the following instructions to set up a dev environment to get started wit pdl examples/demo/1-hello.pdl Hello - Hello! How can I help you today? + Hello ``` ### Documentation updates diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index 11ed762e5..b95607911 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -146,8 +146,11 @@ def __init__(self, monkeypatch: MonkeyPatch) -> None: self.expected_runtime_error = content["expected_runtime_error"] for filename, inputs_type in content["with_inputs"].items(): - stdin = inputs_type["stdin"] - scope = inputs_type["scope"] + stdin, scope = None, None + if "stdin" in inputs_type: + stdin = inputs_type["stdin"] + if "scope" in inputs_type: + scope = inputs_type["scope"] self.with_inputs[filename] = InputsType( stdin=stdin, scope=PdlDict(scope) if scope is not None else None ) @@ -314,10 +317,10 @@ def test_example_runs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> # Print the actual results for wrong results for file, actual in background.failed_results.wrong_results.items(): print( - "============================================================================" + "\n============================================================================" ) print(f"File that produced wrong result: {file}") - print(f"Actual (copy everything below this line):✂️----------------------------------\n{actual}\n-----------------------------------") + print(f"Actual result (copy everything below this line):\n✂️ ------------------------------------------------------------\n{actual}\n-------------------------------------------------------------") assert ( len(background.failed_results.unexpected_parse_error) == 0 diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 3b4c3f23b..0aed7e855 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,4 +1,4 @@ -update_results: true +update_results: false check: [] skip: - examples/cldk/cldk-assistant.pdl @@ -66,6 +66,7 @@ with_inputs: scope: something: ABC tests/data/optimizer_gsm8k.pdl: + stdin: null scope: model: watsonx_text/ibm/granite-3-8b-instruct prompt_pattern: cot From d6927580a0743cc2fe2d4a38b086fb57463d2c7d Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 14:51:38 -0400 Subject: [PATCH 36/37] Update results Signed-off-by: Jing Chen --- docs/contrib.md | 29 +++++++++++++++++++++- tests/results/examples/react/demo.6.result | 23 +++++++++++++++++ tests/test_examples_run.py | 6 +++-- 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 tests/results/examples/react/demo.6.result diff --git a/docs/contrib.md b/docs/contrib.md index fd4b51387..f98f69d93 100644 --- a/docs/contrib.md +++ b/docs/contrib.md @@ -101,7 +101,34 @@ pytest --capture=tee-sys -rfE -s tests/test_examples_run.py --disable-pytest-war A slight variation in the Python version and OS environment can cause a different LLM response, thus Run Examples might fail because it uses exact string matching for PDL outputs. -When you open a pull request (PR) against the `main` branch, a series of status checks will be executed. Specificially, three Run Examples test will be initiated against the files you have added and modified. If there's any variation, you should manually examine the results produced in the Github Actions environment, then copy and paste the results to a new file, and push another commit to your PR so the CI can pass. Be aware of whitespaces in between sentences, as the CI does an exact string matching. +When you open a pull request (PR) against the `main` branch, a series of status checks will be executed. Specificially, three Run Examples test will be initiated against the PDL files you have added and modified as part of the PR. If there's any variation, you should manually examine the results produced in the Github Actions environment, then copy and paste the results to a new file, and push another commit to your PR so the CI can pass. Be aware of whitespaces in between sentences. + +Below is what the CI might look like if the results differ. If the results look reasonable, you should +1. copy and paste the result for that file +2. create a new file in `tests/results` in the format of `tests/results/..result` where `` should be incremented from the highest number of `.result` in the folder + +If the test fails, look for something similar like the following: +``` +tests/test_examples_run.py +============================================================================ +File that produced wrong result: examples/demo/1-hello.pdl +Actual result (copy everything below this line): +✂️ ------------------------------------------------------------ +Hello +Hello +------------------------------------------------------------- + +============================================================================ +File that produced wrong result: examples/demo/2-model-chaining.pdl +Actual result (copy everything below this line): +✂️ ------------------------------------------------------------ +Hello +Hello +Did you just say Hello? +Yes, I did. It's a common greeting, similar to how humans might respond when they first interact with an artificial intelligence like me. How can I assist you today? +------------------------------------------------------------- +F +``` Note: Your PR should always set `update_results: false` before merging, though the CI pipeline will not attempt to write the results to your PR even if you forget to set this field to `false`. diff --git a/tests/results/examples/react/demo.6.result b/tests/results/examples/react/demo.6.result new file mode 100644 index 000000000..cf6b881cb --- /dev/null +++ b/tests/results/examples/react/demo.6.result @@ -0,0 +1,23 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago their birthdate was. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Thought: Henry Hudson discovered the Hudson River in 1609. I need to find out when he was born. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson birthdate"}}] +Observation: Page id "henry hudson birthday" does not match any pages. Try another id! +Thought: I couldn't find a specific birthdate for Henry Hudson, but it's estimated he was born around 1565. + +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Thought: Henry Hudson was born around 1565, so in 2025, it has been approximately 460 years. + +Answer: It has been about 460 years since the birth of Henry Hudson.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] + diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index b95607911..6936147bc 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -146,7 +146,7 @@ def __init__(self, monkeypatch: MonkeyPatch) -> None: self.expected_runtime_error = content["expected_runtime_error"] for filename, inputs_type in content["with_inputs"].items(): - stdin, scope = None, None + stdin, scope = None, None if "stdin" in inputs_type: stdin = inputs_type["stdin"] if "scope" in inputs_type: @@ -320,7 +320,9 @@ def test_example_runs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> "\n============================================================================" ) print(f"File that produced wrong result: {file}") - print(f"Actual result (copy everything below this line):\n✂️ ------------------------------------------------------------\n{actual}\n-------------------------------------------------------------") + print( + f"Actual result (copy everything below this line):\n✂️ ------------------------------------------------------------\n{actual}\n-------------------------------------------------------------" + ) assert ( len(background.failed_results.unexpected_parse_error) == 0 From 87c74fb4417c7da072cda4b7efd0ac5b7db7f841 Mon Sep 17 00:00:00 2001 From: Jing Chen Date: Tue, 3 Jun 2025 15:08:11 -0400 Subject: [PATCH 37/37] Enable parallel runs Signed-off-by: Jing Chen --- .github/workflows/run-examples-prep.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/run-examples-prep.yml b/.github/workflows/run-examples-prep.yml index bc853ca8a..9ea984412 100644 --- a/.github/workflows/run-examples-prep.yml +++ b/.github/workflows/run-examples-prep.yml @@ -7,16 +7,10 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false - max-parallel: 1 matrix: python-version: ['3.11', '3.12', '3.13'] steps: - # - uses: actions/checkout@v4 - # with: - # repository: ${{ github.event.pull_request.head.repo.full_name }} - # ref: ${{ github.event.pull_request.head.ref }} - # Detect modified PDL files, includes Add, Modified, but not Deleted - uses: actions/checkout@v4 with: