Skip to content

Commit 6dd7174

Browse files
jgchnjgchn
committed
Test if demo passes GHA
Signed-off-by: Jing Chen <[email protected]>
1 parent b0f0d8a commit 6dd7174

File tree

5 files changed

+174
-171
lines changed

5 files changed

+174
-171
lines changed

‎examples/intrinsics/demo-hallucination.pdl‎

Lines changed: 132 additions & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -6,172 +6,139 @@
66
# The model responds to a query and provides a hallucination score
77

88
text:
9-
# Builds granite3-rag:8b
10-
# - contribute: [context]
11-
# lang: command
12-
# code: echo "Hello"
9+
# Install dependencies
1310
- contribute: []
1411
lang: command
1512
code: |
1613
[ -d "llama.cpp/.git" ] || git clone https://github.com/ggerganov/llama.cpp.git llama.cpp
17-
# git -C "llama.cpp" pull || git clone https://github.com/ggerganov/llama.cpp.git "llama.cpp"
18-
# [ ! -d 'llama.cpp' ] && git clone https://github.com/ggerganov/llama.cpp.git
19-
# mkdir -p llama.cpp && [[ ! -d "llama.cpp" ]] && git clone --depth 1 https://github.com/ggerganov/llama.cpp.git llama.cpp
20-
# [[ ! -d "llama.cpp" ]] && git clone --depth 1 https://github.com/ggerganov/llama.cpp.git
21-
22-
# # Builds granite3-rag:8b
23-
# - contribute: []
24-
# lang: python
25-
# code: |
26-
# import os
27-
# import shutil
28-
# import subprocess
29-
# import huggingface_hub
30-
# import ollama
31-
32-
# granite3_rag_model = 'granite3-rag:8b'
33-
34-
# # Check if rag model exists
35-
# found = False
36-
# result = ""
37-
# local_ollama_models = ollama.list().models
38-
# for model in local_ollama_models:
39-
# if model.model == granite3_rag_model:
40-
# found = True
41-
# result = f"Model {granite3_rag_model} found in Ollama, skipping build"
42-
# break
43-
44-
# if not found:
45-
# # Install dependencies
46-
# # Define the repository and directory
47-
# llama_cpp_repo = "https://github.com/ggerganov/llama.cpp.git"
48-
# llama_cpp_dir_name = "llama.cpp"
49-
50-
# # Clone the repository if it doesn't already exist
51-
# if not os.path.isdir(llama_cpp_dir_name):
52-
# subprocess.run(["git", "clone", "--depth", "1", llama_cpp_repo], check=True)
53-
54-
# # Install the required packages
55-
# subprocess.run([
56-
# "pip", "install",
57-
# "git+https://github.com/ibm-granite-community/utils.git",
58-
# "huggingface_hub",
59-
# "langchain_community",
60-
# "langchain_ollama",
61-
# "langchain-milvus",
62-
# "docling",
63-
# "-r", f"{llama_cpp_dir_name}/requirements.txt"
64-
# ], check=True)
65-
66-
# # Ollama pull base model
67-
# granite3_base_model = 'granite3-dense:8b'
68-
# ollama.pull(granite3_base_model)
69-
70-
# # HG: download lora adapter
71-
# lora_folder = huggingface_hub.snapshot_download(repo_id="ibm-granite/granite-rag-3.0-8b-lora")
72-
# base_folder = huggingface_hub.snapshot_download(repo_id="ibm-granite/granite-3.0-8b-instruct", allow_patterns="*.json")
73-
# lora_gguf = "granite-rag-3.0-8b-lora-fp16.gguf"
74-
# subprocess.run(f"python3 llama.cpp/convert_lora_to_gguf.py --outtype f16 --outfile {lora_gguf} --base {base_folder} -- {lora_folder}",
75-
# shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
76-
77-
# # Write Modelfile for granite-rag:8b for Ollama
78-
# model_file = "Modelfile"
79-
# with open(model_file, "w") as modelfile:
80-
# modelfile.write(f"""\
81-
# FROM {granite3_base_model}
82-
# ADAPTER {lora_gguf}
83-
# """)
84-
85-
# # Ideally we should use ollama.create() but there is no option to pass a modelfile
86-
# subprocess.run(f"ollama create {granite3_rag_model} -f Modelfile",
87-
# shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
88-
89-
# # Rm dependencies
90-
# os.remove(model_file)
91-
# if os.path.isdir(llama_cpp_dir_name):
92-
# shutil.rmtree(llama_cpp_dir_name)
93-
94-
# result = f"Model {granite3_rag_model} not found on Ollama. Finished building {granite3_rag_model}"
95-
96-
# - role: system
97-
# contribute: [context]
98-
# text:
99-
# data:
100-
# instruction: |
101-
# Respond to the user's latest question based solely on the information provided
102-
# in the documents. Ensure that your response is strictly aligned with the facts
103-
# in the provided documents. If the information needed to answer the question is
104-
# not available in the documents, inform the user that the question cannot be
105-
# answered based on the available data. Make sure that your response follows
106-
# the attributes mentioned in the 'meta' field.
107-
# documents:
108-
# - doc_id: 1
109-
# text: |
110-
# Audrey Faith McGraw (born September 21, 1967) is an American singer
111-
# and record producer. She is one of the most successful country artists
112-
# of all time, having sold more than 40 million albums worldwide. Hill is
113-
# married to American singer Tim McGraw, with whom she has recorded several duets.
114-
# Hill's first two albums, Take Me as I Am (1993) and It Matters to Me (1995),
115-
# were major successes and placed a combined three number ones on Billboard's
116-
# country charts. Hill's debut album was Take Me as I Am (1993); sales were strong,
117-
# buoyed by the chart success of "Wild One". Hill became the first female country
118-
# singer in 30 years to hold Billboard's number one position for four consecutive
119-
# weeks when "Wild One" managed the feat in 1994. Her version of "Piece of My Heart",
120-
# also went to the top of the country charts in 1994. The album sold a total of
121-
# 3 million copies. Other singles from the album include "Take Me as I Am". The recording
122-
# of Faith's second album was delayed by surgery to repair a ruptured blood vessel on
123-
# her vocal cords. It Matters to Me finally appeared in 1995 and was another
124-
# success, with the title track becoming her third number-one country single.
125-
# Several other top 10 singles followed, and more than 3 million copies of the
126-
# album were sold. The fifth single from the album, "I Can't Do That Anymore",
127-
# was written by country music artist Alan Jackson. Other singles from the album
128-
# include "You Can't Lose Me", "Someone Else's Dream", and "Let's Go to Vegas".
129-
# During this period, Hill appeared on the acclaimed PBS music program Austin City Limits.
130-
# In spring 1996, Hill began the Spontaneous Combustion Tour with country singer Tim McGraw.
131-
# At that time, Hill had recently become engaged to her former producer, Scott Hendricks,
132-
# and McGraw had recently broken an engagement. McGraw and Hill were quickly
133-
# attracted to each other and began an affair. After discovering that Hill was
134-
# pregnant with their first child, the couple married on October 6, 1996. The
135-
# couple have three daughters together: Gracie Katherine (born 1997), Maggie Elizabeth (born 1998)
136-
# and Audrey Caroline (born 2001). Since their marriage, Hill and McGraw have endeavored
137-
# never to be apart for more than three consecutive days. After the release of It Matters to Me,
138-
# Hill took a three-year break from recording to give herself a rest from four years of touring
139-
# and to begin a family with McGraw. During her break, she joined forces with her husband
140-
# for their first duet, "It's Your Love". The song stayed at number one for six weeks,
141-
# and won awards from both the Academy of Country Music and the Country Music Association.
142-
# Hill has remarked that sometimes when they perform the song together,
143-
# "it [doesn't] feel like anybody else was really watching."
144-
# meta:
145-
# hallucination_tags: true
146-
# citations: true
147-
148-
# # User Query
149-
# # (This query produces a hallucination "low" with citation)
150-
# - Did Faith Hill take a break from recording after releasing her second album, It Matters to Me?
151-
# # (This query produces a hallucination "unanswerable" with no citation)
152-
# # - Is the Academy of Country Music in Brooklyn, New York?
153-
# # (This query produces a hallucination "high" with a citation)
154-
# # - Where was Faith Hill born?
155-
156-
# # Base model granite3-dense:8b setup with Granite RAG LoRA (Low-Rank Adaption) on ollama.
157-
# - defs:
158-
# # Store the results of making the LLM invocation in a JSON variable named 'out'
159-
# out:
160-
# model: ollama/granite3-rag:8b
161-
# parameters:
162-
# temperature: 0
163-
# parser: json
164-
# - |
165-
166-
167-
# The answer is: ${ out[0].sentence }
168-
# - match: ${out[0].meta.hallucination_level}
169-
# with:
170-
# - case: "high"
171-
# then: Totally hallucinating, sorry!
172-
# - case: "low"
173-
# if: ${ out[0].meta.citation }
174-
# then: |
175-
# I am not hallucinating, promise!
176-
# The citation is: ${ out[0].meta.citation.snippet }
177-
# - then: Not sure if I am hallucinating...
14+
- contribute: []
15+
lang: command
16+
code: |
17+
pip install "git+https://github.com/ibm-granite-community/utils.git" \
18+
"huggingface_hub" \
19+
"langchain_community" \
20+
"langchain_ollama" \
21+
"ollama" \
22+
-r llama.cpp/requirements.txt
23+
# Download base model and LoRA adapter and build granite3-rag model
24+
- contribute: []
25+
lang: python
26+
code: |
27+
import os
28+
import subprocess
29+
import huggingface_hub
30+
import ollama
31+
32+
granite3_model = 'granite3-dense:8b'
33+
granite3_rag_model = 'granite3-rag:8b'
34+
35+
# Ollama pull base model
36+
granite3_base_model = 'granite3-dense:8b'
37+
ollama.pull(granite3_base_model)
38+
39+
# HG: download LoRA adapter
40+
lora_folder = huggingface_hub.snapshot_download(repo_id="ibm-granite/granite-rag-3.0-8b-lora")
41+
base_folder = huggingface_hub.snapshot_download(repo_id="ibm-granite/granite-3.0-8b-instruct", allow_patterns="*.json")
42+
lora_gguf = "granite-rag-3.0-8b-lora-fp16.gguf"
43+
subprocess.run(f"python3 llama.cpp/convert_lora_to_gguf.py --outtype f16 --outfile {lora_gguf} --base {base_folder} -- {lora_folder}",
44+
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
45+
46+
# Write Modelfile for granite-rag:8b for Ollama
47+
model_file = "Modelfile"
48+
with open(model_file, "w") as modelfile:
49+
modelfile.write(f"""\
50+
FROM {granite3_base_model}
51+
ADAPTER {lora_gguf}
52+
""")
53+
54+
# Ideally we should use ollama.create() but there is no option to pass a modelfile
55+
subprocess.run(f"ollama create {granite3_rag_model} -f Modelfile",
56+
shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
57+
58+
# Rm model file
59+
os.remove(model_file)
60+
61+
result = f"Model {granite3_rag_model} available"
62+
63+
- role: system
64+
contribute: [context]
65+
text:
66+
data:
67+
instruction: |
68+
Respond to the user's latest question based solely on the information provided
69+
in the documents. Ensure that your response is strictly aligned with the facts
70+
in the provided documents. If the information needed to answer the question is
71+
not available in the documents, inform the user that the question cannot be
72+
answered based on the available data. Make sure that your response follows
73+
the attributes mentioned in the 'meta' field.
74+
documents:
75+
- doc_id: 1
76+
text: |
77+
Audrey Faith McGraw (born September 21, 1967) is an American singer
78+
and record producer. She is one of the most successful country artists
79+
of all time, having sold more than 40 million albums worldwide. Hill is
80+
married to American singer Tim McGraw, with whom she has recorded several duets.
81+
Hill's first two albums, Take Me as I Am (1993) and It Matters to Me (1995),
82+
were major successes and placed a combined three number ones on Billboard's
83+
country charts. Hill's debut album was Take Me as I Am (1993); sales were strong,
84+
buoyed by the chart success of "Wild One". Hill became the first female country
85+
singer in 30 years to hold Billboard's number one position for four consecutive
86+
weeks when "Wild One" managed the feat in 1994. Her version of "Piece of My Heart",
87+
also went to the top of the country charts in 1994. The album sold a total of
88+
3 million copies. Other singles from the album include "Take Me as I Am". The recording
89+
of Faith's second album was delayed by surgery to repair a ruptured blood vessel on
90+
her vocal cords. It Matters to Me finally appeared in 1995 and was another
91+
success, with the title track becoming her third number-one country single.
92+
Several other top 10 singles followed, and more than 3 million copies of the
93+
album were sold. The fifth single from the album, "I Can't Do That Anymore",
94+
was written by country music artist Alan Jackson. Other singles from the album
95+
include "You Can't Lose Me", "Someone Else's Dream", and "Let's Go to Vegas".
96+
During this period, Hill appeared on the acclaimed PBS music program Austin City Limits.
97+
In spring 1996, Hill began the Spontaneous Combustion Tour with country singer Tim McGraw.
98+
At that time, Hill had recently become engaged to her former producer, Scott Hendricks,
99+
and McGraw had recently broken an engagement. McGraw and Hill were quickly
100+
attracted to each other and began an affair. After discovering that Hill was
101+
pregnant with their first child, the couple married on October 6, 1996. The
102+
couple have three daughters together: Gracie Katherine (born 1997), Maggie Elizabeth (born 1998)
103+
and Audrey Caroline (born 2001). Since their marriage, Hill and McGraw have endeavored
104+
never to be apart for more than three consecutive days. After the release of It Matters to Me,
105+
Hill took a three-year break from recording to give herself a rest from four years of touring
106+
and to begin a family with McGraw. During her break, she joined forces with her husband
107+
for their first duet, "It's Your Love". The song stayed at number one for six weeks,
108+
and won awards from both the Academy of Country Music and the Country Music Association.
109+
Hill has remarked that sometimes when they perform the song together,
110+
"it [doesn't] feel like anybody else was really watching."
111+
meta:
112+
hallucination_tags: true
113+
citations: true
114+
115+
# User Query
116+
# (This query produces a hallucination "low" with citation)
117+
- Did Faith Hill take a break from recording after releasing her second album, It Matters to Me?
118+
# (This query produces a hallucination "unanswerable" with no citation)
119+
# - Is the Academy of Country Music in Brooklyn, New York?
120+
# (This query produces a hallucination "high" with a citation)
121+
# - Where was Faith Hill born?
122+
123+
# Base model granite3-dense:8b setup with Granite RAG LoRA (Low-Rank Adaption) on ollama.
124+
- defs:
125+
# Store the results of making the LLM invocation in a JSON variable named 'out'
126+
out:
127+
model: ollama/granite3-rag:8b
128+
parameters:
129+
temperature: 0
130+
parser: json
131+
- |
132+
133+
134+
The answer is: ${ out[0].sentence }
135+
- match: ${out[0].meta.hallucination_level}
136+
with:
137+
- case: "high"
138+
then: Totally hallucinating, sorry!
139+
- case: "low"
140+
if: ${ out[0].meta.citation }
141+
then: |
142+
I am not hallucinating, promise!
143+
The citation is: ${ out[0].meta.citation.snippet }
144+
- then: Not sure if I am hallucinating...

‎src/pdl/pdl_interpreter.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
# pylint: disable=import-outside-toplevel
22
import json
33
import re
4-
import shlex
54
import subprocess # nosec
65
import sys
76
import time
@@ -1531,9 +1530,12 @@ def call_python(code: str, scope: ScopeType) -> PdlLazy[Any]:
15311530

15321531

15331532
def call_command(code: str) -> PdlLazy[str]:
1534-
args = shlex.split(code)
15351533
p = subprocess.run(
1536-
args, capture_output=True, text=True, check=False, shell=False
1534+
code,
1535+
capture_output=True,
1536+
text=True,
1537+
check=False,
1538+
shell=True
15371539
) # nosec B603
15381540
# [B603:subprocess_without_shell_equals_true] subprocess call - check for execution of untrusted input.
15391541
# This is the code that the user asked to execute. It can be executed in a docker container with the option `--sandbox`

‎tests/results/examples/intrinsics/demo-hallucination.0.result‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Did Faith Hill take a break from recording after releasing her second album, It Matters to Me?
2+
3+
The answer is: Yes, Faith Hill took a three-year break from recording after releasing her second album, It Matters to Me.
4+
I am not hallucinating, promise!
5+
The citation is: After discovering that Hill was
6+
pregnant with their first child, the couple married on October 6, 1996. The
7+
couple have three daughters together: Gracie Katherine (born 1997), Maggie Elizabeth (born 1998)
8+
and Audrey Caroline (born 2001). Since their marriage, Hill and McGraw have endeavored
9+
never to be apart for more than three consecutive days. After the release of It Matters to Me,
10+
Hill took a three-year break from recording to give herself a rest from four years of touring
11+
and to begin a family with McGraw. During her break, she joined forces with her husband
12+
for their first duet, "It's Your Love".
13+

‎tests/results/examples/intrinsics/demo-hallucination.1.result‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Running command git clone --filter=blob:none --quiet https://github.com/ibm-granite-community/utils.git /private/var/folders/jv/ydx4xrxs7jjdzn1qqb50pbs80000gn/T/pip-req-build-pgzu86xo
2+
3+
[notice] A new release of pip is available: 23.2.1 -> 25.0.1
4+
[notice] To update, run: pip install --upgrade pip
5+
6+
Fetching 4 files: 100%|███████████████████████████████████████████████████| 4/4 [00:00<00:00, 30283.78it/s]
7+
Fetching 8 files: 100%|██████████████████████████████████████████████████| 8/8 [00:00<00:00, 112222.18it/s]
8+
Did Faith Hill take a break from recording after releasing her second album, It Matters to Me?
9+
10+
The answer is: Yes, Faith Hill took a three-year break from recording after releasing her second album, It Matters to Me.
11+
I am not hallucinating, promise!
12+
The citation is: After discovering that Hill was
13+
pregnant with their first child, the couple married on October 6, 1996. The
14+
couple have three daughters together: Gracie Katherine (born 1997), Maggie Elizabeth (born 1998)
15+
and Audrey Caroline (born 2001). Since their marriage, Hill and McGraw have endeavored
16+
never to be apart for more than three consecutive days. After the release of It Matters to Me,
17+
Hill took a three-year break from recording to give herself a rest from four years of touring
18+
and to begin a family with McGraw. During her break, she joined forces with her husband
19+
for their first duet, "It's Your Love".

‎tests/test_examples_run.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,6 @@
7373
pathlib.Path("examples") / "chatbot" / "chatbot.pdl", # TODO check why
7474
pathlib.Path("examples") / "fibonacci" / "fib.pdl", # TODO check why
7575
pathlib.Path("examples")
76-
/ "intrinsics"
77-
/ "demo-hallucination.pdl", # TODO check why
78-
pathlib.Path("examples")
7976
/ "hello"
8077
/ "hello-function-empty-context.pdl", # TODO CREATE RESULTS FILE
8178
pathlib.Path("examples") / "hello" / "hello-roles-array.pdl", # TODO check why
@@ -232,7 +229,12 @@ def test_valid_programs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -
232229
actual_parse_error: set[str] = set()
233230
actual_runtime_error: set[str] = set()
234231
wrong_results = {}
232+
233+
test_files = pathlib.Path(".").glob("**/*.pdl")
234+
test_files = pathlib.Path(".").glob("**/demo-halluncination.pdl")
235+
235236
for pdl_file_name in pathlib.Path(".").glob("**/*.pdl"):
237+
# for pdl_file_name in test_files:
236238
scope: ScopeType = PdlDict({})
237239
if str(pdl_file_name) in TO_SKIP:
238240
continue

0 commit comments

Comments
 (0)