GitHub tool (#931)

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation - temporary commit

* TLK-2709 - Github Tool Implementation - temporary commit

* TLK-2709 - Github Tool Implementation - temporary commit

* TLK-2709 - Github Tool Implementation - temporary commit

* TLK-2709 - Github Tool Implementation - temporary commit

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation + small fixes

* TLK-2709 - Github Tool Implementation format

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation

* TLK-2709 - Github Tool Implementation review fixes

* TLK-2709 - Github Tool Implementation review fixes

* TLK-2709 - Github Tool Implementation review fixes

Author: EugeneLightsOn

README.md

@@ -15,6 +15,7 @@ Toolkit is a deployable all-in-one RAG application that enables users to quickly
   - [How to setup Google Drive](/docs/custom_tool_guides/google_drive.md)
   - [How to setup Gmail](/docs/custom_tool_guides/gmail.md)
   - [How to setup Slack Tool](/docs/custom_tool_guides/slack.md)
+  - [How to setup Github Tool](/docs/custom_tool_guides/github.md)
   - [How to setup Google Text-to-Speech](/docs/text_to_speech.md)
   - [How to add authentication](/docs/auth_guide.md)
   - [How to deploy toolkit services](/docs/service_deployments.md)

docs/custom_tool_guides/github.md

@@ -0,0 +1,50 @@
+# Github Tool Setup
+
+To set up the Github tool you will need a Github application. Follow the steps below to set it up:
+
+## 1. Create a Github App
+
+Head to the [Github Settings](https://github.com/settings/apps) and create a new app.
+Specify App [permissions](https://docs.github.com/rest/overview/permissions-required-for-github-apps), Callback URL (for local setup - http://localhost:8000/v1/tool/auth).
+Uncheck the `Webhook->Active` option. After creating the app, you will see the `General` section. Copy the `Client ID`, generate and copy `Client Secret` values.
+That will be used for the environment variables specified below.
+This tool also support OAuth Apps. See the [documentation](https://docs.github.com/en/apps/oauth-apps) for more information.
+
+## 2. Set Up Environment Variables
+Set the configuration in the `configuration.yaml`
+```yaml
+github:
+    default_repos: 
+      - repo1
+      - repo2
+    user_scopes: 
+      - public_repo
+      - read:org
+```
+
+Then set the following secrets variables. You can either set the below values in your `secrets.yaml` file:
+```yaml
+github:
+    client_id: <your_client_id from step 1>
+    client_secret: <your_client_secret from step 1>
+```
+or update your `.env` configuration to contain:
+```dotenv
+GITHUB_CLIENT_ID=<your_client_id from step 1>
+GITHUB_CLIENT_SECRET=<your_client_secret from step 1>
+GITHUB_DEFAULT_REPOS=["repo1","repo2"]
+GITHUB_USER_SCOPES=["public_repo","read:org"]
+```
+Please note if the default repos are not set, the tool will process over all user repos.
+
+## 3. Run the Backend and Frontend
+
+run next command to start the backend and frontend:
+
+```bash
+make dev
+```
+
+## 4. Troubleshooting
+
+If you encounter any issues with OAuth, please check the following [link](https://api.Github.com/authentication/oauth-v2#errors)

poetry.lock

@@ -59,6 +59,7 @@ llama-index-embeddings-cohere = "^0.2.1"
 google-cloud-texttospeech = "^2.18.0"
 slack-sdk = "^3.33.1"
 onnxruntime = "1.19.2"
+pygithub = "^2.5.0"
 
 [tool.poetry.group.dev]
 optional = true

pyproject.toml

@@ -37,6 +37,12 @@ tools:
   slack:
     user_scopes:
       - search:read
+  github:
+    user_scopes:
+      - public_repo
+    default_repos:
+      - cohere-ai/cohere-toolkit
+      - EugeneLightsOn/cohere-toolkit
   # To disable the use of the tools preamble, set it to false
   use_tools_preamble: true
 feature_flags:

src/backend/config/configuration.template.yaml

@@ -37,6 +37,9 @@ tools:
   gmail:
     client_id:
     client_secret:
+  github:
+    client_id:
+    client_secret:
 auth:
   secret_key:
   google_oauth:

src/backend/config/secrets.template.yaml

@@ -191,14 +191,38 @@ class SlackSettings(BaseSettings, BaseModel):
         default=None,
         validation_alias=AliasChoices("SLACK_CLIENT_SECRET", "client_secret"),
     )
-    user_scopes: Optional[str] = Field(
+    user_scopes: Optional[List[str]] = Field(
         default=None,
         validation_alias=AliasChoices(
             "SLACK_USER_SCOPES", "scopes"
         ),
     )
 
 
+class GithubSettings(BaseSettings, BaseModel):
+    model_config = SETTINGS_CONFIG
+    client_id: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("GITHUB_CLIENT_ID", "client_id"),
+    )
+    client_secret: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("GITHUB_CLIENT_SECRET", "client_secret"),
+    )
+    user_scopes: Optional[List[str]] = Field(
+        default=None,
+        validation_alias=AliasChoices(
+            "GITHUB_USER_SCOPES", "user_scopes"
+        ),
+    )
+    default_repos: Optional[List[str]] = Field(
+        default=None,
+        validation_alias=AliasChoices(
+            "GITHUB_DEFAULT_REPOS", "default_repos"
+        ),
+    )
+
+
 class TavilyWebSearchSettings(BaseSettings, BaseModel):
     model_config = SETTINGS_CONFIG
     api_key: Optional[str] = Field(
@@ -272,6 +296,9 @@ class ToolSettings(BaseSettings, BaseModel):
     slack: Optional[SlackSettings] = Field(
         default=SlackSettings()
     )
+    github: Optional[GithubSettings] = Field(
+        default=GithubSettings()
+    )
     gmail: Optional[GmailSettings] = Field(
         default=GmailSettings()
     )

src/backend/config/settings.py

@@ -18,6 +18,7 @@
     TavilyWebSearch,
     WebScrapeTool,
 )
+from backend.tools.github.tool import GithubTool
 
 logger = LoggerFactory().get_logger()
 
@@ -38,6 +39,7 @@ class Tool(Enum):
     Hybrid_Web_Search = HybridWebSearch
     Slack = SlackTool
     Gmail = GmailTool
+    Github = GithubTool
 
 
 def get_available_tools() -> dict[str, ToolDefinition]:

src/backend/config/tools.py

@@ -83,9 +83,10 @@ def create_app() -> FastAPI:
     # Dynamically set router dependencies
     # These values must be set in config/routers.py
     dependencies_type = "default"
+    settings = Settings()
     if is_authentication_enabled():
         # Required to save temporary OAuth state in session
-        auth_secret = Settings().get('auth.secret_key')
+        auth_secret = settings.get('auth.secret_key')
         app.add_middleware(SessionMiddleware, secret_key=auth_secret)
         dependencies_type = "auth"
     for router in routers:

src/backend/main.py

@@ -72,7 +72,7 @@ async def test_wiki_retriever_no_docs() -> None:
     ):
         result = await retriever.call({"query": query}, ctx)
 
-    assert result == ToolError(type=ToolErrorCode.OTHER, success=False, text='No results found.', details='No results found for the given params.')
+    assert result == [ToolError(type=ToolErrorCode.OTHER, success=False, text='No results found.', details='No results found for the given params.').model_dump()]
 
 
 
@@ -156,4 +156,4 @@ async def test_vector_db_retriever_no_docs() -> None:
         mock_db.as_retriever().get_relevant_documents.return_value = mock_docs
         result = await retriever.call({"query": query}, ctx)
 
-    assert result == ToolError(type=ToolErrorCode.OTHER, success=False, text='No results found.', details='No results found for the given params.')
+    assert result == [ToolError(type=ToolErrorCode.OTHER, success=False, text='No results found.', details='No results found for the given params.').model_dump()]