|
| 1 | +import dspy |
| 2 | +from dotenv import find_dotenv, load_dotenv |
| 3 | +from dspy.datasets import HotPotQA |
| 4 | +from dspy.teleprompt import BootstrapFewShot |
| 5 | + |
| 6 | +from langtrace_python_sdk import inject_additional_attributes, langtrace |
| 7 | + |
| 8 | +_ = load_dotenv(find_dotenv()) |
| 9 | + |
| 10 | +langtrace.init() |
| 11 | + |
| 12 | +turbo = dspy.LM('openai/gpt-4o-mini') |
| 13 | +colbertv2_wiki17_abstracts = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts') |
| 14 | + |
| 15 | +dspy.settings.configure(lm=turbo, rm=colbertv2_wiki17_abstracts) |
| 16 | + |
| 17 | + |
| 18 | +# Load the dataset. |
| 19 | +dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0) |
| 20 | + |
| 21 | +# Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata. |
| 22 | +trainset = [x.with_inputs('question') for x in dataset.train] |
| 23 | +devset = [x.with_inputs('question') for x in dataset.dev] |
| 24 | + |
| 25 | + |
| 26 | +class GenerateAnswer(dspy.Signature): |
| 27 | + """Answer questions with short factoid answers.""" |
| 28 | + |
| 29 | + context = dspy.InputField(desc="may contain relevant facts") |
| 30 | + question = dspy.InputField() |
| 31 | + answer = dspy.OutputField(desc="often between 1 and 5 words") |
| 32 | + |
| 33 | + |
| 34 | +class RAG(dspy.Module): |
| 35 | + def __init__(self, num_passages=3): |
| 36 | + super().__init__() |
| 37 | + |
| 38 | + self.retrieve = dspy.Retrieve(k=num_passages) |
| 39 | + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) |
| 40 | + |
| 41 | + def forward(self, question): |
| 42 | + context = self.retrieve(question).passages |
| 43 | + prediction = self.generate_answer(context=context, question=question) |
| 44 | + return dspy.Prediction(context=context, answer=prediction.answer) |
| 45 | + |
| 46 | + |
| 47 | +# Validation logic: check that the predicted answer is correct. |
| 48 | +# Also check that the retrieved context does actually contain that answer. |
| 49 | +def validate_context_and_answer(example, prediction, trace=None): |
| 50 | + answer_em = dspy.evaluate.answer_exact_match(example, prediction) |
| 51 | + answer_pm = dspy.evaluate.answer_passage_match(example, prediction) |
| 52 | + return answer_em and answer_pm |
| 53 | + |
| 54 | + |
| 55 | +# Set up a basic optimizer, which will compile our RAG program. |
| 56 | +optimizer = BootstrapFewShot(metric=validate_context_and_answer) |
| 57 | + |
| 58 | +# Compile! |
| 59 | +compiled_rag = optimizer.compile(RAG(), trainset=trainset) |
| 60 | + |
| 61 | +# Ask any question you like to this simple RAG program. |
| 62 | +my_question = "Who was the hero of the movie peraanmai?" |
| 63 | + |
| 64 | +# Get the prediction. This contains `pred.context` and `pred.answer`. |
| 65 | +# pred = compiled_rag(my_question) |
| 66 | +pred = inject_additional_attributes(lambda: compiled_rag(my_question), {'experiment': 'experiment 6', 'description': 'trying additional stuff', 'run_id': 'run_1'}) |
| 67 | +# compiled_rag.save('compiled_rag_v1.json') |
| 68 | + |
| 69 | +# Print the contexts and the answer. |
| 70 | +print(f"Question: {my_question}") |
| 71 | +print(f"Predicted Answer: {pred.answer}") |
| 72 | +print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}") |
| 73 | + |
| 74 | +# print("Inspecting the history of the optimizer:") |
| 75 | +# turbo.inspect_history(n=1) |
| 76 | + |
| 77 | +from dspy.evaluate import Evaluate |
| 78 | + |
| 79 | + |
| 80 | +def validate_answer(example, pred, trace=None): |
| 81 | + return True |
| 82 | + |
| 83 | + |
| 84 | +# Set up the evaluator, which can be used multiple times. |
| 85 | +evaluate = Evaluate(devset=devset, metric=validate_answer, num_threads=4, display_progress=True, display_table=0) |
| 86 | + |
| 87 | + |
| 88 | +# Evaluate our `optimized_cot` program. |
| 89 | +evaluate(compiled_rag) |
0 commit comments