Merge pull request #13 from openchatai/feat/github-data-source

(wip) Feat/GitHub data source

Author: gharbat

README.md

@@ -35,6 +35,7 @@ https://github.com/openchatai/OpenChat/assets/32633162/c1e0cea1-6627-47c3-becc-c
 - Customize your chatbots by providing PDF files, websites, and soon, integrations with platforms like Notion, Confluence, and Office 365.
 - Each chatbot has unlimited memory capacity, enabling seamless interaction with large files such as a 400-page PDF.
 - Embed chatbots as widgets on your website or internal company tools.
+- Use your entire codebase as a data source for your chatbots (pair programming mode).
 - And much more!
 
 ## 🛣️ Roadmap:
@@ -46,6 +47,7 @@ https://github.com/openchatai/OpenChat/assets/32633162/c1e0cea1-6627-47c3-becc-c
 - [x] Accept websites as a data source
 - [x] Accept PDF files as a data source
 - [x] Support multiple data sources per chatbot
+- [x] Support ingesting an entire codebase using GitHub API and use it as a data source with pair programming mode
 - [ ] Support Slack integration (allow users to connect chatbots with their Slack workspaces)
 - [ ] Support Intercom integration (enable users to sync chat conversations with Intercom)
 - [ ] Support offline open-source models (e.g., Alpaca, LLM drivers)

backend-server/app/Http/Controllers/ChatbotController.php

@@ -3,14 +3,17 @@
 namespace App\Http\Controllers;
 
 use App\Http\Events\ChatbotWasCreated;
+use App\Http\Events\CodebaseDataSourceWasAdded;
 use App\Http\Events\PdfDataSourceWasAdded;
 use App\Http\Requests\CreateChatbotRequest;
+use App\Http\Requests\CreateChatbotViaCodebaseRequest;
 use App\Http\Requests\CreateChatbotViaPdfFlowRequest;
 use App\Http\Requests\SendChatMessageRequest;
 use App\Http\Requests\UpdateCharacterSettingsRequest;
 use App\Http\Responses\ChatbotResponse;
 use App\Http\Services\HandlePdfDataSource;
 use App\Models\Chatbot;
+use App\Models\CodebaseDataSource;
 use Illuminate\Http\JsonResponse;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Support\Facades\Http;
@@ -133,17 +136,19 @@ public function sendMessage(SendChatMessageRequest $request, $token): JsonRespon
         // Get the question and history from the request
         $question = $request->getMessage();
         $history = $request->getHistory();
+        $mode = $request->getMode();
 
         // Remove null and empty values and empty arrays or objects from the history
         $history = array_filter($history, function ($value) {
             return !is_null($value) && $value !== '' && $value !== [] && $value !== (object) [];
         });
 
         // Call the API to send the message to the chatbot with a timeout of 5 seconds
-        $response = Http::connectTimeout(60)->post("http://llm-server:3000/api/chat", [
+        $response = Http::timeout(200)->post("http://llm-server:3000/api/chat", [
             'question' => $question,
             'history' => $history,
             'namespace' => $bot->getId()->toString(),
+            'mode' =>  $mode,
         ]);
 
         if ($response->failed()) {
@@ -178,4 +183,28 @@ public function getChatView($token)
             'bot' => $bot,
         ]);
     }
+
+    public function createViaCodebaseFlow(CreateChatbotViaCodebaseRequest $request): RedirectResponse
+    {
+        $chatbot = new Chatbot();
+        $chatbot->setId(Uuid::uuid4());
+        $chatbot->setName($request->getName());
+        $chatbot->setToken(Str::random(20));
+        $chatbot->setPromptMessage($request->getPromptMessage());
+        $chatbot->save();
+
+        $datasource = new CodebaseDataSource();
+        $datasource->setId(Uuid::uuid4());
+        $datasource->setChatbotId($chatbot->getId());
+        $datasource->setRepository($request->getRepoUrl());
+        $datasource->save();
+
+
+        event(new CodebaseDataSourceWasAdded(
+                $chatbot->getId(),
+                $datasource->getId())
+        );
+
+        return redirect()->route('onboarding.config', ['id' => $chatbot->getId()->toString()]);
+    }
 }

backend-server/app/Http/Controllers/ChatbotSettingController.php

@@ -65,11 +65,13 @@ public function dataSettings(Request $request, $id)
         // Get website data sources and PDF data sources for the chatbot
         $websiteDataSources = $bot->getWebsiteDataSources()->get();
         $pdfDataSources = $bot->getPdfFilesDataSources()->get();
+        $codebaseDataSources = $bot->getCodebaseDataSources()->get();
 
         return view('settings-data', [
             'bot' => $bot,
             'websiteDataSources' => $websiteDataSources,
             'pdfDataSources' => $pdfDataSources,
+            'codebaseDataSources' => $codebaseDataSources,
         ]);
     }
 

backend-server/app/Http/Controllers/OnboardingController.php

@@ -35,4 +35,9 @@ public function done()
     {
         return view('onboarding.step-4');
     }
+
+    public function dataSourcesCodebase()
+    {
+        return view('onboarding.step-2-codebase');
+    }
 }

backend-server/app/Http/Enums/IngestStatusType.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace App\Http\Enums;
+
+class IngestStatusType
+{
+    public const SUCCESS= 'success';
+    public const FAILED = 'failed';
+
+
+    public function __construct(private readonly string $status)
+    {
+        if (!self::isValid($status)) {
+            throw new \InvalidArgumentException("Invalid status: {$status}");
+        }
+    }
+
+    public static function getTypes(): array
+    {
+        return [
+            self::SUCCESS,
+            self::FAILED,
+        ];
+    }
+
+    public static function getLabels(): array
+    {
+        return [
+            self::SUCCESS => 'Success',
+            self::FAILED => 'Failed',
+        ];
+    }
+
+    public function getLabel(): string
+    {
+        return self::getLabels()[$this->getStatus()];
+    }
+
+    public static function getValues(): array
+    {
+        return array_keys(self::getLabels());
+    }
+
+    public static function isValid(string $type): bool
+    {
+        return in_array($type, self::getValues());
+    }
+
+    public function isSuccessful(): bool
+    {
+        return $this->getStatus() === self::SUCCESS;
+    }
+
+    public function isFailed(): bool
+    {
+        return $this->getStatus() === self::FAILED;
+    }
+
+    public function getStatus(): string
+    {
+        return $this->status;
+    }
+
+    public function __toString(): string
+    {
+        return $this->getStatus();
+    }
+
+    public function equals(IngestStatusType $type): bool
+    {
+        return $this->getStatus() === $type->getStatus();
+    }
+}

backend-server/app/Http/Events/CodebaseDataSourceWasAdded.php

@@ -0,0 +1,29 @@
+<?php
+
+namespace App\Http\Events;
+
+use Illuminate\Broadcasting\InteractsWithSockets;
+use Illuminate\Foundation\Events\Dispatchable;
+use Illuminate\Queue\SerializesModels;
+use Ramsey\Uuid\UuidInterface;
+
+class CodebaseDataSourceWasAdded
+{
+    use Dispatchable, InteractsWithSockets, SerializesModels;
+
+    public function __construct(
+        private UuidInterface $chatbotId,
+        private UuidInterface $codebaseDataSourceId,
+    ) {
+    }
+
+    public function getChatbotId(): UuidInterface
+    {
+        return $this->chatbotId;
+    }
+
+    public function getCodebaseDataSourceId(): UuidInterface
+    {
+        return $this->codebaseDataSourceId;
+    }
+}

backend-server/app/Http/Listeners/IngestCodebaseDataSource.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace App\Http\Listeners;
+
+use App\Http\Enums\IngestStatusType;
+use App\Http\Events\CodebaseDataSourceWasAdded;
+use App\Models\CodebaseDataSource;
+use Exception;
+use GuzzleHttp\Client;
+use GuzzleHttp\Exception\GuzzleException;
+use Illuminate\Contracts\Queue\ShouldQueue;
+
+class IngestCodebaseDataSource implements ShouldQueue
+{
+
+    /**
+     * @throws GuzzleException
+     * @throws Exception
+     */
+    public function handle($event)
+    {
+        if (!$event instanceof CodebaseDataSourceWasAdded) {
+            return;
+        }
+
+        $botId = $event->getChatbotId();
+        $codebaseDataSourceId = $event->getCodebaseDataSourceId();
+
+
+        /** @var CodebaseDataSource $datasource */
+        $datasource = CodebaseDataSource::find($codebaseDataSourceId);
+
+
+        $repo = $datasource->getRepository();
+
+        $requestBody = [
+            'type' => 'codebase',
+            'repo' => $repo,
+            'namespace' => $botId,
+        ];
+
+        // Call to ingest service endpoint
+        $client = new Client();
+        $response = $client->request('POST', 'http://llm-server:3000/api/ingest', [
+            'json' => $requestBody,
+        ]);
+
+        $datasource->setIngestedAt(now());
+
+        if ($response->getStatusCode() !== 200) {
+            $datasource->setIngestionStatus(IngestStatusType::FAILED);
+        } else {
+            $datasource->setIngestionStatus(IngestStatusType::SUCCESS);
+        }
+
+        $datasource->save();
+    }
+}

backend-server/app/Http/Requests/CreateChatbotViaCodebaseRequest.php

@@ -0,0 +1,37 @@
+<?php
+
+namespace App\Http\Requests;
+
+
+use App\Http\Rules\GithubRepoUrlRule;
+use Illuminate\Foundation\Http\FormRequest;
+
+class CreateChatbotViaCodebaseRequest extends FormRequest
+{
+    public function rules(): array
+    {
+        return [
+            'repo' => ['required', new GithubRepoUrlRule],
+        ];
+    }
+
+    public function getRepoUrl(): string
+    {
+        return $this->get('repo');
+    }
+
+    public function getRepoAccessToken(): ?string
+    {
+        return $this->get('token');
+    }
+
+    public function getName(): string
+    {
+        return $this->get('name', 'My first chatbot');
+    }
+
+    public function getPromptMessage(): string
+    {
+        return $this->get('prompt_message', "");
+    }
+}

backend-server/app/Http/Requests/SendChatMessageRequest.php

@@ -12,6 +12,7 @@ public function rules(): array
     {
         return [
             'message' => 'required|string',
+            'mode' => 'sometimes|string|in:assistant,pair_programmer',
         ];
     }
 
@@ -24,4 +25,9 @@ public function getHistory(): array
     {
         return $this->get('history');
     }
+
+    public function getMode(): string
+    {
+        return $this->get('mode');
+    }
 }

backend-server/app/Http/Responses/ChatbotResponse.php

@@ -15,6 +15,6 @@ public function getBotReply(): string
 
     public function getSourceDocuments(): array
     {
-        return $this->response['sourceDocuments'];
+        return $this->response['sourceDocuments'] ?? [];
     }
 }