GitHub tool

README.md

@@ -15,6 +15,7 @@ Toolkit is a deployable all-in-one RAG application that enables users to quickly
   - [How to setup Google Drive](/docs/custom_tool_guides/google_drive.md)
   - [How to setup Gmail](/docs/custom_tool_guides/gmail.md)
   - [How to setup Slack Tool](/docs/custom_tool_guides/slack.md)
+  - [How to setup Github Tool](/docs/custom_tool_guides/github.md)
   - [How to setup Google Text-to-Speech](/docs/text_to_speech.md)
   - [How to add authentication](/docs/auth_guide.md)
   - [How to deploy toolkit services](/docs/service_deployments.md)

docs/custom_tool_guides/github.md

@@ -0,0 +1,50 @@
+# Github Tool Setup
+
+To set up the Github tool you will need a Github application. Follow the steps below to set it up:
+
+## 1. Create a Github App
+
+Head to the [Github Settings](https://github.com/settings/apps) and create a new app.
+Specify App [permissions](https://docs.github.com/rest/overview/permissions-required-for-github-apps), Callback URL (for local setup - http://localhost:8000/v1/tool/auth).
+Uncheck the `Webhook->Active` option. After creating the app, you will see the `General` section. Copy the `Client ID`, generate and copy `Client Secret` values.
+That will be used for the environment variables specified above.
+This tool also support OAuth Apps. See the [documentation](https://docs.github.com/en/apps/oauth-apps) for more information.
+
+## 2. Set Up Environment Variables
+Set the configuration in the `configuration.yaml`
+```yaml
+github:
+    default_repos: 
+      - repo1
+      - repo2
+    user_scopes: 
+      - public_repo
+      - read:org
+```
+
+Then set the following secrets variables. You can either set the below values in your `secrets.yaml` file:
+```yaml
+github:
+    client_id: <your_client_id from step 1>
+    client_secret: <your_client_secret from step 1>
+```
+or update your `.env` configuration to contain:
+```dotenv
+GITHUB_CLIENT_ID=<your_client_id from step 1>
+GITHUB_CLIENT_SECRET=<your_client_secret from step 1>
+GITHUB_DEFAULT_REPOS=["repo1","repo2"]
+GITHUB_USER_SCOPES=["public_repo","read:org"]
+```
+Please note if the default repos are not set, the tool will process over all user repos.
+
+## 3. Run the Backend and Frontend
+
+run next command to start the backend and frontend:
+
+```bash
+make dev
+```
+
+## 4. Troubleshooting
+
+If you encounter any issues with OAuth, please check the following [link](https://api.Github.com/authentication/oauth-v2#errors)

pyproject.toml

@@ -59,6 +59,7 @@ llama-index-embeddings-cohere = "^0.2.1"
 google-cloud-texttospeech = "^2.18.0"
 slack-sdk = "^3.33.1"
 onnxruntime = "1.19.2"
+pygithub = "^2.5.0"
 
 [tool.poetry.group.dev]
 optional = true

src/backend/chat/collate.py

@@ -3,6 +3,7 @@
 
 from backend.model_deployments.base import BaseDeployment
 from backend.schemas.context import Context
+from backend.tools.base import ToolError
 
 RELEVANCE_THRESHOLD = 0.1
 
@@ -62,6 +63,10 @@ async def rerank_and_chunk(
 
         chunked_outputs = []
         for output in tool_result["outputs"]:
+            if isinstance(output, ToolError):
+                reranked_results[tool_call_hashable] = tool_result
+                continue
+
             text = output.get("text")
 
             if not text:

src/backend/config/configuration.template.yaml

@@ -37,6 +37,12 @@ tools:
   slack:
     user_scopes:
       - search:read
+  github:
+    user_scopes:
+      - public_repo
+    default_repos:
+      - cohere-ai/cohere-toolkit
+      - EugeneLightsOn/cohere-toolkit
   # To disable the use of the tools preamble, set it to false
   use_tools_preamble: true
 feature_flags:

src/backend/config/default_agent.py

@@ -23,6 +23,7 @@ def get_default_agent() -> AgentPublic:
             Tool.Search_File.value.ID,
             Tool.Python_Interpreter.value.ID,
             Tool.Hybrid_Web_Search.value.ID,
+            Tool.Github.value.ID,
         ],
         tools_metadata=[],
         deployment=DEFAULT_DEPLOYMENT,

src/backend/config/secrets.template.yaml

@@ -37,6 +37,9 @@ tools:
   gmail:
     client_id:
     client_secret:
+  github:
+    client_id:
+    client_secret:
 auth:
   secret_key:
   google_oauth:

src/backend/config/settings.py

@@ -191,14 +191,38 @@ class SlackSettings(BaseSettings, BaseModel):
         default=None,
         validation_alias=AliasChoices("SLACK_CLIENT_SECRET", "client_secret"),
     )
-    user_scopes: Optional[str] = Field(
+    user_scopes: Optional[List[str]] = Field(
         default=None,
         validation_alias=AliasChoices(
             "SLACK_USER_SCOPES", "scopes"
         ),
     )
 
 
+class GithubSettings(BaseSettings, BaseModel):
+    model_config = SETTINGS_CONFIG
+    client_id: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("GITHUB_CLIENT_ID", "client_id"),
+    )
+    client_secret: Optional[str] = Field(
+        default=None,
+        validation_alias=AliasChoices("GITHUB_CLIENT_SECRET", "client_secret"),
+    )
+    user_scopes: Optional[List[str]] = Field(
+        default=None,
+        validation_alias=AliasChoices(
+            "GITHUB_USER_SCOPES", "user_scopes"
+        ),
+    )
+    default_repos: Optional[List[str]] = Field(
+        default=None,
+        validation_alias=AliasChoices(
+            "GITHUB_DEFAULT_REPOS", "default_repos"
+        ),
+    )
+
+
 class TavilyWebSearchSettings(BaseSettings, BaseModel):
     model_config = SETTINGS_CONFIG
     api_key: Optional[str] = Field(
@@ -272,6 +296,9 @@ class ToolSettings(BaseSettings, BaseModel):
     slack: Optional[SlackSettings] = Field(
         default=SlackSettings()
     )
+    github: Optional[GithubSettings] = Field(
+        default=GithubSettings()
+    )
     gmail: Optional[GmailSettings] = Field(
         default=GmailSettings()
     )

src/backend/config/tools.py

@@ -18,6 +18,7 @@
     TavilyWebSearch,
     WebScrapeTool,
 )
+from backend.tools.github.tool import GithubTool
 
 logger = LoggerFactory().get_logger()
 
@@ -38,6 +39,7 @@ class Tool(Enum):
     Hybrid_Web_Search = HybridWebSearch
     Slack = SlackTool
     Gmail = GmailTool
+    Github = GithubTool
 
 
 def get_available_tools() -> dict[str, ToolDefinition]:

src/backend/main.py

@@ -83,9 +83,10 @@ def create_app() -> FastAPI:
     # Dynamically set router dependencies
     # These values must be set in config/routers.py
     dependencies_type = "default"
+    settings = Settings()
     if is_authentication_enabled():
         # Required to save temporary OAuth state in session
-        auth_secret = Settings().get('auth.secret_key')
+        auth_secret = settings.get('auth.secret_key')
         app.add_middleware(SessionMiddleware, secret_key=auth_secret)
         dependencies_type = "auth"
     for router in routers:

src/backend/tools/base.py

@@ -1,7 +1,7 @@
 import datetime
 from abc import ABC, abstractmethod
 from enum import StrEnum
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
 import requests
 from fastapi import Request
@@ -169,7 +169,7 @@ def get_no_results_error(cls):
     @abstractmethod
     async def call(
             self, parameters: dict, ctx: Any, **kwargs: Any
-    ) -> List[Dict[str, Any]]:
+    ) -> Union[List[Dict[str, Any]], ToolError]:
         ...
 
     @classmethod