Skip to content

Commit a894c14

Browse files
MrOrangeAndHamMrOrangeAndHam
committed
Add instructions to create a test and add tests for the repository agent
1 parent fefee4e commit a894c14

File tree

4 files changed

+186
-5
lines changed

4 files changed

+186
-5
lines changed

‎README.md‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,26 @@ Users are allowed to access the application if one of their verified email addre
7474
## Create a tool
7575
7676
Using the [github rest documentation](https://docs.github.com/en/rest), add a new tool.
77-
1. In the `github_tools.py` file, define a method that will be used to leverage the api endpoint you have chosen.
77+
1. In the `github_tools.py` file, define a method that will be used to leverage the API endpoint you have chosen.
7878
2. Within the method, using the python doc format, add query language for openai to use.
7979
3. Add the `@tool()` decorator on the method you just defined.
80-
4. Within the `github_tools.py` file, define a new method for the api call that the tool will leverage to fetch data.
81-
5. Using the new api call, you defined, fetch the data within the tool and return it as a `json dump`.
80+
4. Within the `github_tools.py` file, define a new method for the API call that the tool will leverage to fetch data.
81+
5. Using the new API call, you defined, fetch the data within the tool and return it as a `json dump`.
8282
6. Register the new tool by adding it to the list of tools that are returned by the `github_tools` method.
8383
84-
Run the application and see if you can have openapi use your tool to fetch data.
84+
Run the application and see if you can have openAPI use your tool to fetch data.
85+
86+
## Add a test for the new tool
87+
88+
We want to make sure the Agent integrates properly and uses our tool.
89+
These tests invoke the LLM and take more time to run.
90+
There can be multiple ways of getting a result from an agent so when we are testing our agents
91+
we are focusing on the tools we expect the agent to use is correct and the response is correct.
92+
Test the tool that was added by following these steps:
93+
94+
1. Define a `test_` method that will cover the new tool that was added.
95+
2. Decorate the test method with the `@slow` decorator.
96+
3. Add the `@responses.activate` decorator to the method so we can stub out the API call.
97+
4. Stub the API that the tool you are testing uses.
98+
5. Send a question to the agent and assert the response is what you are expecting.
99+
6. Validate that the tool you think the LLM will use is used.

‎tests/repository_agent/__init__.py‎

Whitespace-only changes.

‎tests/repository_agent/test_repository_agent.py‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import unittest
2+
3+
import responses
4+
from openai import OpenAI
5+
6+
from discovery.agent_support.agent import ToolCall
7+
from discovery.environment import require_env
8+
from discovery.github_support.github_client import GithubClient
9+
from discovery.repository_agent.repository_agent import repository_agent_creator
10+
from tests.slow_test_support import slow
11+
12+
13+
class TestRepositoryAgent(unittest.TestCase):
14+
def setUp(self):
15+
openai_client = OpenAI(api_key=require_env("OPEN_AI_KEY"))
16+
self.repo_dict = {
17+
"name": "pickles_web",
18+
"full_name": "pickles_company/pickles_web",
19+
"html_url": "https://example.com/pickles_company/pickles_web",
20+
"url": "https://api.example.com/pickles_company/pickles_web",
21+
"description": "Pickles Web",
22+
"private": False,
23+
"stargazers_count": 12,
24+
"watchers_count": 13,
25+
"forks_count": 14,
26+
}
27+
28+
self.agent = repository_agent_creator(openai_client)(GithubClient("some-token"))
29+
30+
@slow
31+
@responses.activate
32+
def test_list_repositories_for_organization(self):
33+
responses.add(
34+
responses.GET,
35+
"https://api.github.com/orgs/pickles_company/repos",
36+
json=[self.repo_dict],
37+
status=200,
38+
)
39+
40+
result = self.agent.answer("List repositories for the organization pickles_company.")
41+
42+
self.assertIn("pickles_web", result.response)
43+
self.assertEqual([ToolCall(name="list_repositories_for_organization", arguments={
44+
"organization": "pickles_company"
45+
})], result.tool_calls)
46+
47+
@slow
48+
@responses.activate
49+
def test_list_repositories_for_user(self):
50+
responses.add(
51+
responses.GET,
52+
"https://api.github.com/users/some_user/repos",
53+
json=[self.repo_dict],
54+
status=200,
55+
)
56+
57+
result = self.agent.answer("List repositories for some_user.")
58+
59+
self.assertIn("pickles_web", result.response)
60+
self.assertEqual([ToolCall(name="list_repositories_for_user", arguments={
61+
"user": "some_user"
62+
})], result.tool_calls)
63+
64+
@slow
65+
@responses.activate
66+
def test_search_repositories(self):
67+
responses.add(
68+
responses.GET,
69+
"https://api.github.com/search/repositories",
70+
json={
71+
"items": [
72+
{
73+
"name": "pickles_web",
74+
"full_name": "pickles_company/pickles_web",
75+
"html_url": "https://example.com/pickles_company/pickles_web",
76+
"url": "https://api.example.com/pickles_company/pickles_web",
77+
"description": "Pickles Web",
78+
"private": False,
79+
"stargazers_count": 12,
80+
"watchers_count": 13,
81+
"forks_count": 14,
82+
},
83+
{
84+
"name": "chickens_web",
85+
"full_name": "pickles_company/dill_pickles_web",
86+
"html_url": "https://example.com/pickles_company/dill_pickles_web",
87+
"url": "https://api.example.com/pickles_company/dill_pickles_web",
88+
"description": "Dill Pickles Web",
89+
"private": True,
90+
"stargazers_count": 12,
91+
"watchers_count": 13,
92+
"forks_count": 15
93+
}
94+
]
95+
},
96+
status=200,
97+
)
98+
99+
result = self.agent.answer(
100+
"Search the repositories that are private for the pickles_company and tell me the name of the repository")
101+
102+
self.assertIn("dill_pickles_web", result.response)
103+
self.assertEqual(
104+
[ToolCall(
105+
name='search_repositories',
106+
arguments={'owner': 'pickles_company', 'owner_type': 'org', 'query': 'private'}
107+
)],
108+
result.tool_calls
109+
)
110+
111+
@slow
112+
@responses.activate
113+
def test_list_repository_languages(self):
114+
responses.add(
115+
responses.GET,
116+
"https://api.github.com/repos/pickles_org/pickles_repo/languages",
117+
json={
118+
"python": 100,
119+
"html": 50,
120+
"css": 50,
121+
},
122+
status=200,
123+
)
124+
125+
result = self.agent.answer("What languages does the 'pickles_repo' from the 'pickles_org' use?")
126+
127+
self.assertIn("Python", result.response)
128+
self.assertIn("HTML", result.response)
129+
self.assertIn("CSS", result.response)
130+
self.assertNotIn("java", result.response)
131+
self.assertEqual(
132+
[ToolCall(
133+
name='list_repository_languages',
134+
arguments={'full_name': 'pickles_org/pickles_repo'}
135+
)],
136+
result.tool_calls
137+
)
138+
139+
@slow
140+
@responses.activate
141+
def test_list_repository_contributors(self):
142+
responses.add(
143+
responses.GET,
144+
"https://api.github.com/repos/pickles_org/pickles_repo/contributors",
145+
json=[
146+
{"login": "fred"},
147+
{"login": "mary"},
148+
{"login": "kate"},
149+
],
150+
status=200,
151+
)
152+
153+
result = self.agent.answer("Who contributes to the pickles_repo within the pickles_org?")
154+
155+
self.assertIn("fred", result.response)
156+
self.assertIn("mary", result.response)
157+
self.assertIn("kate", result.response)
158+
self.assertNotIn("chuck", result.response)
159+
self.assertEqual(
160+
[ToolCall(
161+
name='list_repository_contributors',
162+
arguments={'full_name': 'pickles_org/pickles_repo'}
163+
)],
164+
result.tool_calls
165+
)

‎tests/test_index_page.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66

77

88
class FakeAgent:
9-
def answer(self, question: str) -> AgentResult:
9+
@staticmethod
10+
def answer(_: str) -> AgentResult:
1011
return AgentResult(
1112
response="Some response",
1213
tool_calls=[

0 commit comments

Comments
 (0)