Skip to content

Commit abbb794

Browse files
jgchnjgchn
committed
Test withnew pipline
Signed-off-by: Jing Chen <[email protected]>
1 parent 1a90386 commit abbb794

File tree

4 files changed

+496
-71
lines changed

4 files changed

+496
-71
lines changed

‎.github/workflows/run-examples-modified.yml‎

Lines changed: 80 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ name: Run examples on modified PDL files
33

44
on: [push, pull_request]
55

6-
76
jobs:
87
tests:
98
name: Execution tests
@@ -15,49 +14,49 @@ jobs:
1514
python-version: ['3.11']
1615

1716
steps:
18-
# Free up some disk space
19-
- name: Remove unnecessary files
20-
run: |
21-
sudo rm -rf /usr/share/dotnet
22-
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
17+
# # Free up some disk space
18+
# - name: Remove unnecessary files
19+
# run: |
20+
# sudo rm -rf /usr/share/dotnet
21+
# sudo rm -rf "$AGENT_TOOLSDIRECTORY"
2322

24-
# Set up Ollama
25-
- name: Install Ollama and start server
26-
shell: bash
27-
run: |
28-
curl -fsSL https://ollama.com/install.sh | sudo -E sh
23+
# # Set up Ollama
24+
# - name: Install Ollama and start server
25+
# shell: bash
26+
# run: |
27+
# curl -fsSL https://ollama.com/install.sh | sudo -E sh
2928

30-
- name: Pull models in examples/
31-
shell: bash
32-
run: |
33-
ollama pull granite3.2:2b
34-
ollama pull granite3.2:8b
35-
ollama pull mxbai-embed-large
36-
ollama list
29+
# - name: Pull models in examples/
30+
# shell: bash
31+
# run: |
32+
# ollama pull granite3.2:2b
33+
# ollama pull granite3.2:8b
34+
# ollama pull mxbai-embed-large
35+
# ollama list
3736

38-
- name: Check that all required models are available
39-
shell: bash
40-
run: |
41-
models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b")
42-
missing=0
43-
for model in "${models[@]}"; do
44-
if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then
45-
echo "❌ Model $model (or substring) is missing!"
46-
missing=1
47-
fi
48-
done
37+
# - name: Check that all required models are available
38+
# shell: bash
39+
# run: |
40+
# models=("mxbai-embed-large" "granite3.2:2b" "granite3.2:8b")
41+
# missing=0
42+
# for model in "${models[@]}"; do
43+
# if ! ollama list | awk 'NR>1 {print $1}' | grep -q "$model"; then
44+
# echo "❌ Model $model (or substring) is missing!"
45+
# missing=1
46+
# fi
47+
# done
4948

50-
if [ "$missing" -eq 1 ]; then
51-
exit 1
52-
else
53-
echo "✅ All expected models are available."
54-
fi
49+
# if [ "$missing" -eq 1 ]; then
50+
# exit 1
51+
# else
52+
# echo "✅ All expected models are available."
53+
# fi
5554

56-
- name: Wait for Ollama server
57-
shell: bash
58-
run: |
59-
sleep 10
60-
time curl -i http://localhost:11434
55+
# - name: Wait for Ollama server
56+
# shell: bash
57+
# run: |
58+
# sleep 10
59+
# time curl -i http://localhost:11434
6160

6261
# Run tests
6362
- uses: actions/checkout@v4
@@ -69,38 +68,49 @@ jobs:
6968
with:
7069
files: |
7170
**.pdl
72-
- name: List PDL files that were modified or added
71+
json: 'true'
72+
- name: List PDL files that were modified or added and append to test_examples_run
7373
env:
7474
MODIFIED_PDL_FILES: ${{ steps.changed-pdl-files.outputs.all_changed_files }}
7575
run: echo "$MODIFIED_PDL_FILES"
76-
- name: Set up Python ${{ matrix.python-version }}
77-
uses: actions/setup-python@v5
78-
with:
79-
python-version: ${{ matrix.python-version }}
80-
- name: Cache pip
81-
uses: actions/cache@v4
76+
- name: Update tests/test_example_run.yaml
77+
uses: fjogeleit/yaml-update-action@main
8278
with:
83-
# This path is specific to Ubuntu
84-
path: ${{ env.pythonLocation }}
85-
# Look to see if there is a cache hit for the setup file
86-
key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
87-
restore-keys: |
88-
${{ runner.os }}-pip-new3
89-
${{ runner.os }}-new3
90-
- name: Install dependencies
91-
run: pip install --upgrade --upgrade-strategy eager .[all]
92-
- name: pip list packages
93-
run: pip list
94-
- name: show pip dependencies
95-
run: |
96-
pip install pipdeptree
97-
pipdeptree -fl
98-
- name: run tests
99-
env:
100-
WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
101-
WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
102-
WATSONX_URL: ${{ secrets.WATSONX_URL }}
103-
REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}
104-
OLLAMA_GHACTIONS_RESULTS: true
105-
MODIFIED_PDL_FILES: ${{ steps.changed-pdl-files.outputs.all_changed_files }}
106-
run: py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py
79+
valueFile: 'tests/test_example_run.yaml'
80+
propertyPath: 'check'
81+
value: "f${{ fromJSON(steps.changed-pdl-files.outputs.all_changed_files) }}"
82+
commitChange: false
83+
84+
- name: print yaml config
85+
run: cat tests/test_example_run.yaml
86+
87+
88+
# - name: Set up Python ${{ matrix.python-version }}
89+
# uses: actions/setup-python@v5
90+
# with:
91+
# python-version: ${{ matrix.python-version }}
92+
# - name: Cache pip
93+
# uses: actions/cache@v4
94+
# with:
95+
# # This path is specific to Ubuntu
96+
# path: ${{ env.pythonLocation }}
97+
# # Look to see if there is a cache hit for the setup file
98+
# key: ${{ runner.os }}-pip-new3-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}
99+
# restore-keys: |
100+
# ${{ runner.os }}-pip-new3
101+
# ${{ runner.os }}-new3
102+
# - name: Install dependencies
103+
# run: pip install --upgrade --upgrade-strategy eager .[all]
104+
# - name: pip list packages
105+
# run: pip list
106+
# - name: show pip dependencies
107+
# run: |
108+
# pip install pipdeptree
109+
# pipdeptree -fl
110+
# - name: run tests
111+
# env:
112+
# WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
113+
# WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
114+
# WATSONX_URL: ${{ secrets.WATSONX_URL }}
115+
# REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}
116+
# run: py.test -v --capture=tee-sys -rfE -s tests/test_examples_run.py

‎examples/chatbot/chatbot.pdl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ description: Chatbot
22
text:
33
# Allow the user to type any question, implicitly adding the question to the context.
44
- read:
5-
message: "What is your query?\n"
5+
message: "What is your query??\n"
66
- repeat:
77
text:
88
# Send context to Granite model hosted at ollama

‎tests/test_examples_run.yaml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# Configuration file for test_examples_run.py
2+
update_results: False
3+
results_version: 1
4+
5+
# A subset of files to check, useful for local dev
6+
# If empty, check all files
7+
# check accepts a list of strings
8+
check:
9+
# - examples/chatbot/chatbot.pdl
10+
# - examples/tutorial/programs/chatbot.pdl # with inputs
11+
# - examples/rag/pdf_index.pdl # skip
12+
# - tests/data/line/hello3.pdl # parse error
13+
# - examples/callback/repair_prompt.pdl # runtime error
14+
15+
# Files to skip
16+
skip:
17+
# Requires dataset dependency
18+
- examples/cldk/cldk-assistant.pdl
19+
- examples/gsm8k/gsm8.pdl
20+
- examples/gsm8k/gsm8k-plan.pdl
21+
22+
# Requires installation dependencies
23+
- examples/intrinsics/demo-hallucination.pdl
24+
- examples/tutorial/programs/demo-hallucination.pdl
25+
26+
# Skip RAG examples
27+
- examples/rag/pdf_index.pdl
28+
- examples/rag/pdf_query.pdl
29+
- examples/rag/rag_library1.pdl
30+
31+
# Skip structure decoding examples for now
32+
- examples/tutorial/structured_decoding.pdl
33+
34+
# Output result include trace (and thus timing) for some reason. Investigate why
35+
- examples/react/react_call.pdl
36+
37+
# UI examples
38+
- pdl-live-react/demos/error.pdl
39+
- pdl-live-react/demos/demo1.pdl
40+
- pdl-live-react/demos/demo2.pdl
41+
42+
# Granite-io examples
43+
- examples/granite-io/granite_io_hallucinations.pdl
44+
- examples/granite-io/granite_io_openai.pdl
45+
- examples/granite-io/granite_io_thinking.pdl
46+
- examples/granite-io/granite_io_transformers.pdl
47+
48+
# Files that require input
49+
with_inputs:
50+
examples/tutorial/programs/chatbot.pdl:
51+
stdin: "What is APR?\nyes\n"
52+
scope: null
53+
examples/chatbot/chatbot.pdl:
54+
stdin: "What is APR?\nyes\n"
55+
scope: null
56+
examples/demo/7-chatbot-roles.pdl:
57+
stdin: "What is APR?\nquit\n"
58+
scope: null
59+
examples/tutorial/input_stdin.pdl:
60+
stdin: "What is APR?\nyes\n"
61+
scope: null
62+
examples/tutorial/input_stdin_multiline.pdl:
63+
stdin: "Hello\nBye\n"
64+
scope: null
65+
examples/input/input_test1.pdl:
66+
stdin: "Hello\n"
67+
scope: null
68+
examples/input/input_test2.pdl:
69+
stdin: "Hello\n"
70+
scope: null
71+
examples/tutorial/free_variables.pdl:
72+
stdin: null
73+
scope: {
74+
"something": "ABC"
75+
}
76+
77+
# Files expecting PDL parse error
78+
expected_parse_error:
79+
- tests/data/line/hello.pdl
80+
- tests/data/line/hello1.pdl
81+
- tests/data/line/hello4.pdl
82+
- tests/data/line/hello7.pdl
83+
- tests/data/line/hello8.pdl
84+
- tests/data/line/hello10.pdl
85+
- tests/data/line/hello11.pdl
86+
- tests/data/line/hello31.pdl
87+
88+
# Files expecting runtime error
89+
expected_runtime_error:
90+
- examples/callback/repair_prompt.pdl
91+
- examples/tutorial/type_list.pdl
92+
- examples/tutorial/type_checking.pdl
93+
- tests/data/line/hello3.pdl
94+
- tests/data/line/hello9.pdl
95+
- tests/data/line/hello12.pdl
96+
- tests/data/line/hello13.pdl
97+
- tests/data/line/hello14.pdl
98+
- tests/data/line/hello15.pdl
99+
- tests/data/line/hello16.pdl
100+
- tests/data/line/hello17.pdl
101+
- tests/data/line/hello18.pdl
102+
- tests/data/line/hello19.pdl
103+
- tests/data/line/hello20.pdl
104+
- tests/data/line/hello21.pdl
105+
- tests/data/line/hello22.pdl
106+
- tests/data/line/hello23.pdl
107+
- tests/data/line/hello24.pdl
108+
- tests/data/line/hello25.pdl
109+
- tests/data/line/hello26.pdl
110+
- tests/data/line/hello27.pdl
111+
- tests/data/line/hello28.pdl
112+
- tests/data/line/hello29.pdl
113+
- tests/data/line/hello30.pdl

0 commit comments

Comments
 (0)