Skip to content

Commit 6510bd8

Browse files
authored
refactor(prompts): prompt templates (#66)
1 parent a4fd411 commit 6510bd8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+1200
-1081
lines changed

benchmark/dbally_benchmark/e2e_benchmark.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
import dbally
2424
from dbally.collection import Collection
2525
from dbally.collection.exceptions import NoViewFoundError
26-
from dbally.iql_generator.iql_prompt_template import UnsupportedQueryError, default_iql_template
26+
from dbally.iql_generator.prompt import IQL_GENERATION_TEMPLATE, UnsupportedQueryError
2727
from dbally.llms.litellm import LiteLLM
28-
from dbally.view_selection.view_selector_prompt_template import default_view_selector_template
28+
from dbally.view_selection.prompt import VIEW_SELECTION_TEMPLATE
2929

3030

3131
async def _run_dbally_for_single_example(example: BIRDExample, collection: Collection) -> Text2SQLResult:
@@ -126,9 +126,9 @@ async def evaluate(cfg: DictConfig) -> Any:
126126
logger.info(f"db-ally predictions saved under directory: {output_dir}")
127127

128128
if run:
129-
run["config/iql_prompt_template"] = stringify_unsupported(default_iql_template.chat)
130-
run["config/view_selection_prompt_template"] = stringify_unsupported(default_view_selector_template.chat)
131-
run["config/iql_prompt_template"] = stringify_unsupported(default_iql_template)
129+
run["config/iql_prompt_template"] = stringify_unsupported(IQL_GENERATION_TEMPLATE.chat)
130+
run["config/view_selection_prompt_template"] = stringify_unsupported(VIEW_SELECTION_TEMPLATE.chat)
131+
run["config/iql_prompt_template"] = stringify_unsupported(IQL_GENERATION_TEMPLATE)
132132
run[f"evaluation/{metrics_file_name}"].upload((output_dir / metrics_file_name).as_posix())
133133
run[f"evaluation/{results_file_name}"].upload((output_dir / results_file_name).as_posix())
134134
run["evaluation/metrics"] = stringify_unsupported(metrics)

benchmark/dbally_benchmark/iql_benchmark.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,8 @@
2121

2222
from dbally.audit.event_tracker import EventTracker
2323
from dbally.iql_generator.iql_generator import IQLGenerator
24-
from dbally.iql_generator.iql_prompt_template import UnsupportedQueryError, default_iql_template
24+
from dbally.iql_generator.prompt import IQL_GENERATION_TEMPLATE, UnsupportedQueryError
2525
from dbally.llms.litellm import LiteLLM
26-
from dbally.prompts.formatters import IQLInputFormatter
2726
from dbally.views.structured import BaseStructuredView
2827

2928

@@ -32,14 +31,17 @@ async def _run_iql_for_single_example(
3231
) -> IQLResult:
3332
filter_list = view.list_filters()
3433
event_tracker = EventTracker()
35-
input_formatter = IQLInputFormatter(question=example.question, filters=filter_list)
3634

3735
try:
38-
iql_filters, _ = await iql_generator.generate_iql(input_formatter=input_formatter, event_tracker=event_tracker)
36+
iql_filters = await iql_generator.generate_iql(
37+
question=example.question,
38+
filters=filter_list,
39+
event_tracker=event_tracker,
40+
)
3941
except UnsupportedQueryError:
4042
return IQLResult(question=example.question, iql_filters="UNSUPPORTED_QUERY", exception_raised=True)
4143

42-
return IQLResult(question=example.question, iql_filters=iql_filters, exception_raised=False)
44+
return IQLResult(question=example.question, iql_filters=str(iql_filters), exception_raised=False)
4345

4446

4547
async def run_iql_for_dataset(
@@ -139,7 +141,7 @@ async def evaluate(cfg: DictConfig) -> Any:
139141
logger.info(f"IQL predictions saved under directory: {output_dir}")
140142

141143
if run:
142-
run["config/iql_prompt_template"] = stringify_unsupported(default_iql_template.chat)
144+
run["config/iql_prompt_template"] = stringify_unsupported(IQL_GENERATION_TEMPLATE.chat)
143145
run[f"evaluation/{metrics_file_name}"].upload((output_dir / metrics_file_name).as_posix())
144146
run[f"evaluation/{results_file_name}"].upload((output_dir / results_file_name).as_posix())
145147
run["evaluation/metrics"] = stringify_unsupported(metrics)

benchmark/dbally_benchmark/text2sql/prompt_template.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from dbally.prompts import PromptTemplate
1+
from dbally.prompt import PromptTemplate
22

33
TEXT2SQL_PROMPT_TEMPLATE = PromptTemplate(
44
(

docs/about/roadmap.md

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,13 @@ Below you can find a list of planned features and integrations.
1010
## Planned Features
1111

1212
- [ ] **Support analytical queries**: support for exposing operations beyond filtering.
13-
- [ ] **Few-shot prompting configuration**: allow users to configure the few-shot prompting in View definition to
13+
- [x] **Few-shot prompting configuration**: allow users to configure the few-shot prompting in View definition to
1414
improve IQL generation accuracy.
1515
- [ ] **Request contextualization**: allow to provide extra context for db-ally runs, such as user asking the question.
1616
- [X] **OpenAI Assistants API adapter**: allow to embed db-ally into OpenAI's Assistants API to easily extend the
1717
capabilities of the assistant.
1818
- [ ] **Langchain adapter**: allow to embed db-ally into Langchain applications.
1919

20-
2120
## Integrations
2221

2322
Being agnostic to the underlying technology is one of the main goals of db-ally.

docs/how-to/llms/custom.md

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -44,42 +44,29 @@ class MyLLMClient(LLMClient[LiteLLMOptions]):
4444

4545
async def call(
4646
self,
47-
prompt: ChatFormat,
48-
response_format: Optional[Dict[str, str]],
47+
conversation: ChatFormat,
4948
options: LiteLLMOptions,
5049
event: LLMEvent,
50+
json_mode: bool = False,
5151
) -> str:
5252
# Your LLM API call
5353
```
5454

55-
The [`call`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient.call) method is an abstract method that must be implemented in your subclass. This method should call the LLM inference API and return the response.
55+
The [`call`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient.call) method is an abstract method that must be implemented in your subclass. This method should call the LLM inference API and return the response in string format.
5656

5757
### Step 3: Use tokenizer to count tokens
5858

59-
The [`count_tokens`](../../reference/llms/index.md#dbally.llms.base.LLM.count_tokens) method is used to count the number of tokens in the messages. You can override this method in your custom class to use the tokenizer and count tokens specifically for your model.
59+
The [`count_tokens`](../../reference/llms/index.md#dbally.llms.base.LLM.count_tokens) method is used to count the number of tokens in the prompt. You can override this method in your custom class to use the tokenizer and count tokens specifically for your model.
6060

6161
```python
6262
class MyLLM(LLM[LiteLLMOptions]):
6363

64-
def count_tokens(self, messages: ChatFormat, fmt: Dict[str, str]) -> int:
65-
# Count tokens in the messages in a custom way
64+
def count_tokens(self, prompt: PromptTemplate) -> int:
65+
# Count tokens in the prompt in a custom way
6666
```
6767
!!!warning
6868
Incorrect token counting can cause problems in the [`NLResponder`](../../reference/nl_responder.md#dbally.nl_responder.nl_responder.NLResponder) and force the use of an explanation prompt template that is more generic and does not include specific rows from the IQL response.
6969

70-
### Step 4: Define custom prompt formatting
71-
72-
The [`format_prompt`](../../reference/llms/index.md#dbally.llms.base.LLM.format_prompt) method is used to apply formatting to the prompt template. You can override this method in your custom class to change how the formatting is performed.
73-
74-
```python
75-
class MyLLM(LLM[LiteLLMOptions]):
76-
77-
def format_prompt(self, template: PromptTemplate, fmt: Dict[str, str]) -> ChatFormat:
78-
# Apply custom formatting to the prompt template
79-
```
80-
!!!note
81-
In general, implementation of this method is not required unless the LLM API does not support [OpenAI conversation formatting](https://platform.openai.com/docs/api-reference/chat/create#chat-create-messages){:target="_blank"}. If your model API expects a different format, override this method to avoid issues with inference call.
82-
8370
## Customising LLM Options
8471

8572
[`LLMOptions`](../../reference/llms/index.md#dbally.llms.clients.base.LLMOptions) is a class that defines the options your LLM will use. To create a custom options, you need to create a subclass of [`LLMOptions`](../../reference/llms/index.md#dbally.llms.clients.base.LLMOptions) and define the required properties that will be passed to the [`LLMClient`](../../reference/llms/index.md#dbally.llms.clients.base.LLMClient).

docs/how-to/views/few-shots.md

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# How-To: Define few shots
2+
3+
There are many ways to improve the accuracy of IQL generation - one of them is to use few-shot prompting. db-ally allows you to inject few-shot examples for any type of defined view, both structured and freeform.
4+
5+
Few shots are defined in the [`list_few_shots`](../../reference/views/index.md#dbally.views.base.BaseView.list_few_shots) method, each few shot example should be an instance of [`FewShotExample`](../../reference/prompt.md#dbally.prompt.elements.FewShotExample) class that defines example question and expected LLM answer.
6+
7+
## Structured views
8+
9+
For structured views, both questions and answers for [`FewShotExample`](../../reference/prompt.md#dbally.prompt.elements.FewShotExample) can be defined as a strings, whereas in case of answers Python expressions are also allowed (please see lambda function in example below).
10+
11+
```python
12+
from dbally.prompt.elements import FewShotExample
13+
from dbally.views.sqlalchemy_base import SqlAlchemyBaseView
14+
15+
class RecruitmentView(SqlAlchemyBaseView):
16+
"""
17+
A view for retrieving candidates from the database.
18+
"""
19+
20+
def list_few_shots(self) -> List[FewShotExample]:
21+
return [
22+
FewShotExample(
23+
"Which candidates studied at University of Toronto?",
24+
'studied_at("University of Toronto")',
25+
),
26+
FewShotExample(
27+
"Do we have any soon available perfect fits for senior data scientist positions?",
28+
lambda: (
29+
self.is_available_within_months(1)
30+
and self.data_scientist_position()
31+
and self.has_seniority("senior")
32+
),
33+
),
34+
...
35+
]
36+
```
37+
38+
## Freeform views
39+
40+
Currently freeform views accept SQL query syntax as a raw string. The larger variety of passing parameters is considered to be implemented in further db-ally releases.
41+
42+
```python
43+
from dbally.prompt.elements import FewShotExample
44+
from dbally.views.freeform.text2sql import BaseText2SQLView
45+
46+
class RecruitmentView(BaseText2SQLView):
47+
"""
48+
A view for retrieving candidates from the database.
49+
"""
50+
51+
def list_few_shots(self) -> List[FewShotExample]:
52+
return [
53+
FewShotExample(
54+
"Which candidates studied at University of Toronto?",
55+
'SELECT name FROM candidates WHERE university = "University of Toronto"',
56+
),
57+
FewShotExample(
58+
"Which clients are from NY?",
59+
'SELECT name FROM clients WHERE city = "NY"',
60+
),
61+
...
62+
]
63+
```
64+
65+
## Prompt format
66+
67+
By default each few shot is injected subsequent to a system prompt message. The format is as follows:
68+
69+
```python
70+
[
71+
{
72+
"role" "user",
73+
"content": "Question",
74+
},
75+
{
76+
"role": "assistant",
77+
"content": "Answer",
78+
}
79+
]
80+
```
81+
82+
If you use `examples` formatting tag in content field of the system or user message, all examples are going to be injected inside the message without additional conversation.
83+
84+
The example of prompt utilizing `examples` tag:
85+
86+
```python
87+
[
88+
{
89+
"role" "system",
90+
"content": "Here are example resonses:\n {examples}",
91+
},
92+
]
93+
```
94+
95+
!!!info
96+
There is no best way to inject a few shot example. Different models can behave diffrently based on few shots formatting of choice.
97+
Generally, first appoach should yield the best results in most cases. Therefore, adding example tags in your custom prompts is not recommended.

docs/reference/collection.md

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
!!! tip
44
To understand the general idea better, visit the [Collection concept page](../concepts/collections.md).
55

6-
::: dbally.create_collection
7-
86
::: dbally.collection.Collection
97

108
::: dbally.collection.results.ExecutionResult

docs/reference/index.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
# dbally
22

3-
43
::: dbally.create_collection
Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
11
# IQLGenerator
22

33
::: dbally.iql_generator.iql_generator.IQLGenerator
4-
5-
::: dbally.iql_generator.iql_prompt_template.IQLPromptTemplate
6-
7-
::: dbally.iql_generator.iql_prompt_template.default_iql_template

docs/reference/nl_responder.md

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,3 @@ Otherwise, a response is generated using a `nl_responder_prompt_template`.
2626
To understand general idea better, visit the [NL Responder concept page](../concepts/nl_responder.md).
2727

2828
::: dbally.nl_responder.nl_responder.NLResponder
29-
30-
::: dbally.nl_responder.query_explainer_prompt_template
31-
32-
::: dbally.nl_responder.nl_responder_prompt_template.default_nl_responder_template

0 commit comments

Comments
 (0)