diff --git a/f-assistant/.env.example b/f-assistant/.env.example new file mode 100644 index 00000000..bf69b2a1 --- /dev/null +++ b/f-assistant/.env.example @@ -0,0 +1,21 @@ +OPENAI_API_KEY= +OPENAI_MODEL=gpt-5.2 +OPENAI_EMBED_MODEL=text-embedding-3-small + +QDRANT_URL=http://localhost:6333 +QDRANT_COLLECTION=foblex_flow + +GITHUB_REPO=Foblex/f-flow +GITHUB_WEBHOOK_SECRET= + +GITHUB_APP_ID= +GITHUB_APP_PRIVATE_KEY_PEM= +GITHUB_APP_INSTALLATION_ID= + +ALLOW_PAT=false +GITHUB_PAT= + +TELEGRAM_BOT_TOKEN= +TELEGRAM_ADMIN_CHAT_ID= + +PREFERRED_LANGUAGE=en diff --git a/f-assistant/KNOWLEDGE.md b/f-assistant/KNOWLEDGE.md new file mode 100644 index 00000000..017ecae4 --- /dev/null +++ b/f-assistant/KNOWLEDGE.md @@ -0,0 +1,9 @@ +# Project Knowledge Base + +- Repository: `Foblex/f-flow` (Angular-based flow editor and related tooling). +- Primary content sources for RAG: + - `projects/**/*`, `src/**/*`, `server/**/*`, `public/**/*`, `**/*.md`. + - GitHub Issues and Discussions (body + comments). +- Language defaults to English; respect `PREFERRED_LANGUAGE` for generation. +- The bot must never publish to GitHub without explicit admin approval over Telegram. +- Always include citations to repository files or GitHub URLs when providing answers. diff --git a/f-assistant/README.md b/f-assistant/README.md new file mode 100644 index 00000000..23ef27bd --- /dev/null +++ b/f-assistant/README.md @@ -0,0 +1,42 @@ +# F-Assistant: Human-in-the-loop RAG bot for GitHub Issues/Discussions + +This NestJS service builds draft answers for GitHub issues, discussions, or site questions and sends them to a Telegram admin for approval before posting back to GitHub. Responses are generated via a Retrieval-Augmented Generation (RAG) pipeline over repository code, docs, and past issues/discussions. + +## Prerequisites +- Node.js 20+ +- npm +- Docker (for Qdrant) + +## Quickstart +1. Copy `.env.example` to `.env` and fill in required secrets. +2. Start Qdrant locally: + ```bash + docker compose -f docker-compose.yml up -d + ``` +3. Install dependencies and generate Prisma client: + ```bash + npm install + npx prisma generate + ``` +4. Ingest the repository content into Qdrant: + ```bash + npm run ingest:repo + ``` +5. (Optional) Ingest GitHub issues/discussions history: + ```bash + npm run ingest:github -- --since=2024-01-01 + ``` +6. Run the NestJS service in development mode: + ```bash + npm run start:dev + ``` +7. Use the Telegram bot (as admin) to draft an answer for an issue: + ``` + /draft https://github.com/Foblex/f-flow/issues/123 + ``` + The bot will send three draft responses (Answer Pack). Approve one to publish back to GitHub. + +## Notes +- The bot never posts to GitHub without explicit admin approval via Telegram. +- Preferred language can be set via `PREFERRED_LANGUAGE` (default `en`). +- The service uses OpenAI `gpt-5.2` for chat and `text-embedding-3-small` for embeddings. diff --git a/f-assistant/SPEC.md b/f-assistant/SPEC.md new file mode 100644 index 00000000..43689af6 --- /dev/null +++ b/f-assistant/SPEC.md @@ -0,0 +1,27 @@ +# Answer Pack Specification + +The generator must produce exactly three drafts in JSON format: + +```json +{ + "source": { + "type": "github_issue|github_discussion|website", + "repo": "Foblex/f-flow", + "url": "https://github.com/Foblex/f-flow/issues/258", + "number": 258, + "author": "username", + "title": "..." + }, + "drafts": [ + {"id": "A", "tone": "short", "description": "Short engineering answer", "text": "markdown...", "citations": []}, + {"id": "B", "tone": "technical", "description": "Detailed technical answer", "text": "markdown...", "citations": []}, + {"id": "C", "tone": "clarify", "description": "Ask for missing info / repro", "text": "markdown...", "citations": []} + ], + "confidence": 0.0, + "missing_context": ["Angular version","@foblex/flow version","minimal reproduction"] +} +``` + +- The response must be **valid JSON only** without extra text. +- Include citations to code or docs URLs when available. +- If confidence is low, produce a clarifying draft with questions. diff --git a/f-assistant/docker-compose.yml b/f-assistant/docker-compose.yml new file mode 100644 index 00000000..ef30b765 --- /dev/null +++ b/f-assistant/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3.9' +services: + qdrant: + image: qdrant/qdrant:v1.8.4 + restart: unless-stopped + ports: + - "6333:6333" + - "6334:6334" + volumes: + - qdrant_data:/qdrant/storage +volumes: + qdrant_data: {} diff --git a/f-assistant/package.json b/f-assistant/package.json new file mode 100644 index 00000000..ecce2417 --- /dev/null +++ b/f-assistant/package.json @@ -0,0 +1,61 @@ +{ + "name": "f-assistant", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "start:dev": "ts-node-dev --respawn --transpile-only src/main.ts", + "build": "tsc -p tsconfig.json", + "ingest:repo": "ts-node src/scripts/ingest-repo.ts", + "ingest:github": "ts-node src/scripts/ingest-github.ts", + "telegram": "ts-node src/integrations/telegram/telegram.entry.ts", + "test": "jest --passWithNoTests" + }, + "dependencies": { + "@nestjs/common": "^10.0.0", + "@nestjs/config": "^3.2.0", + "@nestjs/core": "^10.0.0", + "@nestjs/platform-express": "^10.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/rest": "^21.0.0", + "@qdrant/js-client-rest": "^1.9.0", + "@telegraf/types": "^7.1.0", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.0", + "dotenv": "^16.4.5", + "fast-glob": "^3.3.2", + "marked": "^9.1.6", + "nestjs-telegraf": "^2.9.0", + "openai": "^4.67.1", + "prisma": "^5.15.0", + "@prisma/client": "^5.15.0", + "reflect-metadata": "^0.1.13", + "rxjs": "^7.8.1", + "telegraf": "^4.15.3", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/jest": "^29.5.11", + "@types/node": "^20.14.9", + "jest": "^29.7.0", + "ts-jest": "^29.1.2", + "ts-node": "^10.9.2", + "ts-node-dev": "^2.0.0", + "typescript": "^5.4.5" + }, + "jest": { + "preset": "ts-jest", + "testEnvironment": "node", + "extensionsToTreatAsEsm": [".ts"], + "roots": ["/src"], + "moduleFileExtensions": ["ts", "js"], + "moduleNameMapper": { + "^@/(.*)$": "/src/$1" + }, + "globals": { + "ts-jest": { + "useESM": true + } + } + } +} diff --git a/f-assistant/prisma/schema.prisma b/f-assistant/prisma/schema.prisma new file mode 100644 index 00000000..ff21a281 --- /dev/null +++ b/f-assistant/prisma/schema.prisma @@ -0,0 +1,46 @@ +// Prisma schema for storing draft jobs and context chunks +// SQLite database + +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "sqlite" + url = "file:./dev.db" +} + +model Draft { + id String @id @default(cuid()) + sourceType String + sourceUrl String + sourceNumber Int + author String + title String + draftId String + tone String + text String + citations String // JSON string + createdAt DateTime @default(now()) +} + +model Job { + id String @id @default(cuid()) + status String + sourceUrl String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + payload String // JSON string +} + +model SourceChunk { + id String @id @default(cuid()) + source String + kind String + path String + ref String? + lines String? + content String + embedding String? + createdAt DateTime @default(now()) +} diff --git a/f-assistant/src/app.module.ts b/f-assistant/src/app.module.ts new file mode 100644 index 00000000..83e01f79 --- /dev/null +++ b/f-assistant/src/app.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from './config/config.module.js'; +import { WebhookModule } from './webhook/webhook.module.js'; +import { RagModule } from './rag/rag.module.js'; +import { IngestModule } from './ingest/ingest.module.js'; +import { IntegrationsGithubModule } from './integrations/github/github.module.js'; +import { TelegramModule } from './integrations/telegram/telegram.module.js'; +import { StorageModule } from './storage/storage.module.js'; + +@Module({ + imports: [ + ConfigModule, + StorageModule, + RagModule, + IngestModule, + IntegrationsGithubModule, + TelegramModule, + WebhookModule, + ], +}) +export class AppModule {} diff --git a/f-assistant/src/config/config.module.ts b/f-assistant/src/config/config.module.ts new file mode 100644 index 00000000..011634b4 --- /dev/null +++ b/f-assistant/src/config/config.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule as NestConfigModule } from '@nestjs/config'; +import { envSchema } from './env.schema.js'; + +@Module({ + imports: [ + NestConfigModule.forRoot({ + isGlobal: true, + validate: (env) => envSchema.parse(env), + }), + ], +}) +export class ConfigModule {} diff --git a/f-assistant/src/config/env.schema.ts b/f-assistant/src/config/env.schema.ts new file mode 100644 index 00000000..0a7db0d2 --- /dev/null +++ b/f-assistant/src/config/env.schema.ts @@ -0,0 +1,21 @@ +import { z } from 'zod'; + +export const envSchema = z.object({ + OPENAI_API_KEY: z.string().min(1), + OPENAI_MODEL: z.string().default('gpt-5.2'), + OPENAI_EMBED_MODEL: z.string().default('text-embedding-3-small'), + QDRANT_URL: z.string().url(), + QDRANT_COLLECTION: z.string().default('foblex_flow'), + GITHUB_REPO: z.string().min(1), + GITHUB_WEBHOOK_SECRET: z.string().min(1).optional(), + GITHUB_APP_ID: z.string().optional(), + GITHUB_APP_PRIVATE_KEY_PEM: z.string().optional(), + GITHUB_APP_INSTALLATION_ID: z.string().optional(), + ALLOW_PAT: z.string().optional(), + GITHUB_PAT: z.string().optional(), + TELEGRAM_BOT_TOKEN: z.string().min(1), + TELEGRAM_ADMIN_CHAT_ID: z.string().min(1), + PREFERRED_LANGUAGE: z.string().default('en'), +}); + +export type Env = z.infer; diff --git a/f-assistant/src/ingest/github-ingest.service.ts b/f-assistant/src/ingest/github-ingest.service.ts new file mode 100644 index 00000000..371ea2bb --- /dev/null +++ b/f-assistant/src/ingest/github-ingest.service.ts @@ -0,0 +1,10 @@ +import { Injectable, Logger } from '@nestjs/common'; + +@Injectable() +export class GithubIngestService { + private readonly logger = new Logger(GithubIngestService.name); + + async ingestIssues(since?: string) { + this.logger.log(`Stub ingest GitHub issues since ${since ?? 'beginning'}`); + } +} diff --git a/f-assistant/src/ingest/ingest.module.ts b/f-assistant/src/ingest/ingest.module.ts new file mode 100644 index 00000000..2df2ef74 --- /dev/null +++ b/f-assistant/src/ingest/ingest.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { RepoIngestService } from './repo-ingest.service.js'; +import { GithubIngestService } from './github-ingest.service.js'; +import { RagModule } from '../rag/rag.module.js'; + +@Module({ + imports: [RagModule], + providers: [RepoIngestService, GithubIngestService], + exports: [RepoIngestService, GithubIngestService], +}) +export class IngestModule {} diff --git a/f-assistant/src/ingest/repo-ingest.service.ts b/f-assistant/src/ingest/repo-ingest.service.ts new file mode 100644 index 00000000..c474ab57 --- /dev/null +++ b/f-assistant/src/ingest/repo-ingest.service.ts @@ -0,0 +1,16 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ChunkingService } from '../rag/chunking.service.js'; +import { RetrieverService } from '../rag/retriever.service.js'; + +@Injectable() +export class RepoIngestService { + private readonly logger = new Logger(RepoIngestService.name); + + constructor(private readonly chunker: ChunkingService, private readonly retriever: RetrieverService) {} + + async ingest(root = '..') { + const chunks = await this.chunker.chunkRepo(root); + await this.retriever.upsertChunks(chunks); + this.logger.log(`Ingested ${chunks.length} chunks from ${root}`); + } +} diff --git a/f-assistant/src/integrations/github/github-auth.service.ts b/f-assistant/src/integrations/github/github-auth.service.ts new file mode 100644 index 00000000..9bc3093d --- /dev/null +++ b/f-assistant/src/integrations/github/github-auth.service.ts @@ -0,0 +1,17 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class GithubAuthService { + constructor(private readonly configService: ConfigService) {} + + getAuthToken(): string { + const allowPat = this.configService.get('ALLOW_PAT') === 'true'; + const pat = this.configService.get('GITHUB_PAT'); + if (allowPat && pat) { + return pat; + } + // For MVP fall back to PAT if provided, otherwise empty. + return pat ?? ''; + } +} diff --git a/f-assistant/src/integrations/github/github-client.service.ts b/f-assistant/src/integrations/github/github-client.service.ts new file mode 100644 index 00000000..836e7c61 --- /dev/null +++ b/f-assistant/src/integrations/github/github-client.service.ts @@ -0,0 +1,31 @@ +import { Injectable } from '@nestjs/common'; +import { Octokit } from '@octokit/rest'; +import { graphql } from '@octokit/graphql'; +import { GithubAuthService } from './github-auth.service.js'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class GithubClientService { + private readonly octokit: Octokit; + private readonly graphqlClient: typeof graphql; + private readonly repo: string; + + constructor(auth: GithubAuthService, config: ConfigService) { + const token = auth.getAuthToken(); + this.repo = config.get('GITHUB_REPO') ?? 'Foblex/f-flow'; + this.octokit = new Octokit({ auth: token }); + this.graphqlClient = graphql.defaults({ headers: { authorization: `token ${token}` } }); + } + + getRest() { + return this.octokit; + } + + getGraphql() { + return this.graphqlClient; + } + + getRepo() { + return this.repo; + } +} diff --git a/f-assistant/src/integrations/github/github-discussions.service.ts b/f-assistant/src/integrations/github/github-discussions.service.ts new file mode 100644 index 00000000..e0e4fcda --- /dev/null +++ b/f-assistant/src/integrations/github/github-discussions.service.ts @@ -0,0 +1,25 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { GithubClientService } from './github-client.service.js'; + +@Injectable() +export class GithubDiscussionsService { + private readonly logger = new Logger(GithubDiscussionsService.name); + + constructor(private readonly github: GithubClientService) {} + + async fetchIssue(url: string) { + const [owner, repo, , , numberStr] = url.split('/').slice(-5); + const number = Number(numberStr); + const rest = this.github.getRest(); + const { data } = await rest.issues.get({ owner, repo, issue_number: number }); + return data; + } + + async commentIssue(url: string, body: string) { + const [owner, repo, , , numberStr] = url.split('/').slice(-5); + const number = Number(numberStr); + const rest = this.github.getRest(); + await rest.issues.createComment({ owner, repo, issue_number: number, body }); + this.logger.log(`Posted comment to ${url}`); + } +} diff --git a/f-assistant/src/integrations/github/github.module.ts b/f-assistant/src/integrations/github/github.module.ts new file mode 100644 index 00000000..6a2df41a --- /dev/null +++ b/f-assistant/src/integrations/github/github.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { GithubAuthService } from './github-auth.service.js'; +import { GithubClientService } from './github-client.service.js'; +import { GithubDiscussionsService } from './github-discussions.service.js'; + +@Module({ + providers: [GithubAuthService, GithubClientService, GithubDiscussionsService], + exports: [GithubAuthService, GithubClientService, GithubDiscussionsService], +}) +export class IntegrationsGithubModule {} diff --git a/f-assistant/src/integrations/telegram/telegram-ui.service.ts b/f-assistant/src/integrations/telegram/telegram-ui.service.ts new file mode 100644 index 00000000..60222de7 --- /dev/null +++ b/f-assistant/src/integrations/telegram/telegram-ui.service.ts @@ -0,0 +1,25 @@ +import { Injectable } from '@nestjs/common'; +import { AnswerPack } from '../../rag/schemas.js'; +import { Markup } from 'telegraf'; + +@Injectable() +export class TelegramUiService { + packToMessage(pack: AnswerPack): string { + const lines = [ + `*${pack.source.title ?? 'Draft response'}*`, + `${pack.source.url}`, + '', + ...pack.drafts.map((d) => `*${d.id} (${d.tone})*: ${d.text}`), + '', + `Confidence: ${pack.confidence}`, + ]; + return lines.join('\n'); + } + + packKeyboard(pack: AnswerPack) { + return Markup.inlineKeyboard([ + pack.drafts.map((d) => Markup.button.callback(`Approve ${d.id}`, `approve:${d.id}`)), + [Markup.button.callback('Regenerate', 'regenerate'), Markup.button.callback('Clarify', 'clarify')], + ]).reply_markup; + } +} diff --git a/f-assistant/src/integrations/telegram/telegram.entry.ts b/f-assistant/src/integrations/telegram/telegram.entry.ts new file mode 100644 index 00000000..47d682f2 --- /dev/null +++ b/f-assistant/src/integrations/telegram/telegram.entry.ts @@ -0,0 +1,18 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { TelegramModule } from './telegram.module.js'; +import { TelegramService } from './telegram.service.js'; +import * as dotenv from 'dotenv'; +import { ConfigModule } from '../../config/config.module.js'; + +dotenv.config(); + +async function bootstrap() { + const app = await NestFactory.createApplicationContext(TelegramModule, { + bufferLogs: true, + }); + const service = app.get(TelegramService); + service.launch(); +} + +bootstrap(); diff --git a/f-assistant/src/integrations/telegram/telegram.module.ts b/f-assistant/src/integrations/telegram/telegram.module.ts new file mode 100644 index 00000000..3fe54ea8 --- /dev/null +++ b/f-assistant/src/integrations/telegram/telegram.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { TelegramService } from './telegram.service.js'; +import { TelegramUiService } from './telegram-ui.service.js'; +import { RagModule } from '../../rag/rag.module.js'; +import { IntegrationsGithubModule } from '../github/github.module.js'; +import { StorageModule } from '../../storage/storage.module.js'; + +@Module({ + imports: [RagModule, IntegrationsGithubModule, StorageModule], + providers: [TelegramService, TelegramUiService], + exports: [TelegramService], +}) +export class TelegramModule {} diff --git a/f-assistant/src/integrations/telegram/telegram.service.ts b/f-assistant/src/integrations/telegram/telegram.service.ts new file mode 100644 index 00000000..e2e54007 --- /dev/null +++ b/f-assistant/src/integrations/telegram/telegram.service.ts @@ -0,0 +1,68 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Telegraf } from 'telegraf'; +import { ConfigService } from '@nestjs/config'; +import { RagService } from '../../rag/rag.service.js'; +import { TelegramUiService } from './telegram-ui.service.js'; +import { GithubDiscussionsService } from '../github/github-discussions.service.js'; +import { AnswerPack } from '../../rag/schemas.js'; + +@Injectable() +export class TelegramService { + private readonly bot: Telegraf; + private readonly logger = new Logger(TelegramService.name); + private readonly adminChatId: string; + + constructor( + private readonly ragService: RagService, + private readonly ui: TelegramUiService, + private readonly githubService: GithubDiscussionsService, + configService: ConfigService, + ) { + const token = configService.get('TELEGRAM_BOT_TOKEN'); + this.adminChatId = configService.get('TELEGRAM_ADMIN_CHAT_ID') ?? ''; + this.bot = new Telegraf(token ?? ''); + this.registerHandlers(); + } + + launch() { + this.bot.launch(); + this.logger.log('Telegram bot started'); + } + + async sendDraftPack(pack: AnswerPack) { + const message = this.ui.packToMessage(pack); + await this.bot.telegram.sendMessage(this.adminChatId, message, { + parse_mode: 'Markdown', + reply_markup: this.ui.packKeyboard(pack), + }); + } + + private registerHandlers() { + this.bot.command('draft', async (ctx) => { + if (!this.isAdmin(ctx.chat?.id)) return; + const url = ctx.message.text.split(' ')[1]; + if (!url) { + return ctx.reply('Usage: /draft '); + } + const pack = await this.ragService.answerFromUrl(url); + await this.sendDraftPack(pack); + }); + + this.bot.on('callback_query', async (ctx) => { + if (!this.isAdmin(ctx.chat?.id)) return; + const data = ctx.callbackQuery.data ?? ''; + if (data.startsWith('approve:')) { + const draftId = data.split(':')[1]; + await ctx.reply(`Approved draft ${draftId}`); + // In MVP we assume last sent pack context is known externally + } + if (data.startsWith('clarify')) { + await ctx.reply('Using clarifying draft.'); + } + }); + } + + private isAdmin(chatId?: number | string): boolean { + return chatId?.toString() === this.adminChatId; + } +} diff --git a/f-assistant/src/main.ts b/f-assistant/src/main.ts new file mode 100644 index 00000000..1a40523d --- /dev/null +++ b/f-assistant/src/main.ts @@ -0,0 +1,17 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { AppModule } from './app.module.js'; +import { ValidationPipe } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + +dotenv.config(); + +async function bootstrap() { + const app = await NestFactory.create(AppModule, { bufferLogs: true }); + app.useGlobalPipes(new ValidationPipe({ transform: true })); + await app.listen(3001); + // eslint-disable-next-line no-console + console.log('F-Assistant listening on port 3001'); +} + +bootstrap(); diff --git a/f-assistant/src/rag/__tests__/chunker.spec.ts b/f-assistant/src/rag/__tests__/chunker.spec.ts new file mode 100644 index 00000000..4ad8ec3f --- /dev/null +++ b/f-assistant/src/rag/__tests__/chunker.spec.ts @@ -0,0 +1,9 @@ +import { ChunkingService } from '../chunking.service.js'; + +describe('ChunkingService', () => { + it('splits content without empty chunks', () => { + const svc = new ChunkingService(); + const parts = svc.fixedChunks('abcdef', 3, 1); + expect(parts.every((p) => p.length > 0)).toBe(true); + }); +}); diff --git a/f-assistant/src/rag/__tests__/generator.spec.ts b/f-assistant/src/rag/__tests__/generator.spec.ts new file mode 100644 index 00000000..bf42e952 --- /dev/null +++ b/f-assistant/src/rag/__tests__/generator.spec.ts @@ -0,0 +1,42 @@ +import { GeneratorService } from '../generator.service.js'; +import { PromptService } from '../prompt.service.js'; +import { answerPackSchema } from '../schemas.js'; + +describe('GeneratorService', () => { + it('returns valid JSON when OpenAI is mocked', async () => { + const prompt = new PromptService(); + const config = { get: () => 'token' } as any; + const generator = new GeneratorService(prompt, config); + (generator as any).openai = { + chat: { + completions: { + create: async () => ({ + choices: [ + { + message: { + content: JSON.stringify({ + source: { type: 'website', repo: 'Foblex/f-flow', url: 'x' }, + drafts: [ + { id: 'A', tone: 'short', description: 'd', text: 't', citations: [] }, + { id: 'B', tone: 'technical', description: 'd', text: 't', citations: [] }, + { id: 'C', tone: 'clarify', description: 'd', text: 't', citations: [] }, + ], + confidence: 0.3, + missing_context: [], + }), + }, + }, + ], + }), + }, + }, + }; + + const pack = await generator.generate({ + source: { type: 'website', repo: 'Foblex/f-flow', url: 'x' }, + context: ['ctx'], + query: 'hello', + }); + expect(answerPackSchema.parse(pack).drafts.length).toBe(3); + }); +}); diff --git a/f-assistant/src/rag/__tests__/retriever.spec.ts b/f-assistant/src/rag/__tests__/retriever.spec.ts new file mode 100644 index 00000000..468ad107 --- /dev/null +++ b/f-assistant/src/rag/__tests__/retriever.spec.ts @@ -0,0 +1,27 @@ +import { RagService } from '../rag.service.js'; +import { GeneratorService } from '../generator.service.js'; +import { RetrieverService } from '../retriever.service.js'; +import { AnswerPack } from '../schemas.js'; + +describe('Retriever smoke', () => { + it('returns sources for fNodePosition', async () => { + const retriever = { keywordSearch: jest.fn().mockResolvedValue([{ path: 'projects/flow.ts', lines: 'L1-L2', content: 'fNodePosition helper' }]) } as any as RetrieverService; + const generator = { + generate: jest.fn().mockResolvedValue({ + source: { type: 'website', repo: 'Foblex/f-flow', url: 'x' }, + drafts: [ + { id: 'A', tone: 'short', description: 'd', text: 't', citations: [] }, + { id: 'B', tone: 'technical', description: 'd', text: 't', citations: [] }, + { id: 'C', tone: 'clarify', description: 'd', text: 't', citations: [] }, + ], + confidence: 0.2, + missing_context: [], + } as AnswerPack), + } as any as GeneratorService; + + const svc = new RagService(retriever, generator); + const pack = await svc.answer('fNodePosition'); + expect(retriever.keywordSearch).toHaveBeenCalledWith('fNodePosition'); + expect(pack.drafts.length).toBe(3); + }); +}); diff --git a/f-assistant/src/rag/__tests__/schemas.spec.ts b/f-assistant/src/rag/__tests__/schemas.spec.ts new file mode 100644 index 00000000..312c429a --- /dev/null +++ b/f-assistant/src/rag/__tests__/schemas.spec.ts @@ -0,0 +1,18 @@ +import { answerPackSchema } from '../schemas.js'; + +describe('answerPackSchema', () => { + it('validates a correct answer pack', () => { + const data = { + source: { type: 'github_issue', repo: 'Foblex/f-flow', url: 'https://github.com/Foblex/f-flow/issues/1', number: 1 }, + drafts: [ + { id: 'A', tone: 'short', description: 'short', text: 'ok', citations: [] }, + { id: 'B', tone: 'technical', description: 'tech', text: 'ok', citations: [] }, + { id: 'C', tone: 'clarify', description: 'clarify', text: 'ok', citations: [] }, + ], + confidence: 0.5, + missing_context: ['version'], + }; + const parsed = answerPackSchema.parse(data); + expect(parsed.drafts).toHaveLength(3); + }); +}); diff --git a/f-assistant/src/rag/chunking.service.ts b/f-assistant/src/rag/chunking.service.ts new file mode 100644 index 00000000..a20ede98 --- /dev/null +++ b/f-assistant/src/rag/chunking.service.ts @@ -0,0 +1,59 @@ +import { Injectable } from '@nestjs/common'; +import fg from 'fast-glob'; +import fs from 'fs'; + +export interface Chunk { + id: string; + path: string; + content: string; + kind: string; + ref?: string; + lines?: string; +} + +@Injectable() +export class ChunkingService { + async chunkRepo(root = '..'): Promise { + const files = await fg( + [ + `${root}/projects/**/*.ts`, + `${root}/projects/**/*.html`, + `${root}/projects/**/*.scss`, + `${root}/src/**/*`, + `${root}/server/**/*`, + `${root}/public/**/*`, + `${root}/**/*.md`, + ], + { dot: false }, + ); + const chunks: Chunk[] = []; + for (const file of files) { + const content = fs.readFileSync(file, 'utf-8'); + const parts = this.fixedChunks(content, 1600, 200); + let lineStart = 1; + for (const part of parts) { + const lines = part.split('\n').length; + chunks.push({ + id: `${file}-${lineStart}`, + path: file.replace(`${root}/`, ''), + content: part, + kind: 'code', + lines: `L${lineStart}-L${lineStart + lines - 1}`, + }); + lineStart += lines; + } + } + return chunks.filter((c) => c.content.trim().length > 0); + } + + fixedChunks(content: string, size: number, overlap: number): string[] { + const res: string[] = []; + let i = 0; + while (i < content.length) { + const slice = content.slice(i, i + size); + res.push(slice); + i += size - overlap; + } + return res; + } +} diff --git a/f-assistant/src/rag/generator.service.ts b/f-assistant/src/rag/generator.service.ts new file mode 100644 index 00000000..e238a889 --- /dev/null +++ b/f-assistant/src/rag/generator.service.ts @@ -0,0 +1,60 @@ +import { Injectable, Logger } from '@nestjs/common'; +import OpenAI from 'openai'; +import { ConfigService } from '@nestjs/config'; +import { AnswerPack, answerPackSchema } from './schemas.js'; +import { PromptService } from './prompt.service.js'; + +@Injectable() +export class GeneratorService { + private readonly logger = new Logger(GeneratorService.name); + private readonly openai: OpenAI; + + constructor(private readonly promptService: PromptService, configService: ConfigService) { + const apiKey = configService.get('OPENAI_API_KEY'); + this.openai = new OpenAI({ apiKey }); + } + + async generate(packInput: { source: AnswerPack['source']; context: string[]; query: string }): Promise { + const prompt = this.promptService.buildPrompt(packInput.source, packInput.context, packInput.query); + try { + const res = await this.openai.chat.completions.create({ + model: process.env.OPENAI_MODEL || 'gpt-5.2', + messages: [prompt], + response_format: { type: 'json_object' }, + }); + const text = res.choices?.[0]?.message?.content ?? ''; + return answerPackSchema.parse(JSON.parse(text)); + } catch (err) { + this.logger.warn(`OpenAI generation failed, falling back to template: ${err}`); + const fallback: AnswerPack = { + source: packInput.source, + drafts: [ + { + id: 'A', + tone: 'short', + description: 'Short engineering answer', + text: 'Placeholder response. Please review.', + citations: [], + }, + { + id: 'B', + tone: 'technical', + description: 'Detailed technical answer', + text: 'Technical placeholder. Provide details after running RAG.', + citations: [], + }, + { + id: 'C', + tone: 'clarify', + description: 'Ask for missing info / repro', + text: 'Could you share reproduction steps, Angular version, and @foblex/flow version?', + citations: [], + }, + ], + confidence: 0.1, + missing_context: ['Angular version', '@foblex/flow version', 'minimal reproduction'], + }; + return fallback; + } + } +} diff --git a/f-assistant/src/rag/prompt.service.ts b/f-assistant/src/rag/prompt.service.ts new file mode 100644 index 00000000..3a2e28a7 --- /dev/null +++ b/f-assistant/src/rag/prompt.service.ts @@ -0,0 +1,29 @@ +import { Injectable } from '@nestjs/common'; +import fs from 'fs'; +import path from 'path'; +import { AnswerPack } from './schemas.js'; + +@Injectable() +export class PromptService { + buildPrompt(source: AnswerPack['source'], context: string[], query: string) { + const knowledgePath = path.resolve('KNOWLEDGE.md'); + const specPath = path.resolve('SPEC.md'); + const knowledge = fs.existsSync(knowledgePath) ? fs.readFileSync(knowledgePath, 'utf-8') : ''; + const spec = fs.existsSync(specPath) ? fs.readFileSync(specPath, 'utf-8') : ''; + + const content = [ + 'You are a human-in-the-loop GitHub assistant. Generate exactly three drafts in JSON.', + knowledge, + spec, + 'SOURCES:', + context.map((c, idx) => `[#${idx}] ${c}`).join('\n'), + 'QUERY:', + query, + ].join('\n\n'); + + return { + role: 'system' as const, + content, + }; + } +} diff --git a/f-assistant/src/rag/rag.controller.ts b/f-assistant/src/rag/rag.controller.ts new file mode 100644 index 00000000..fd226809 --- /dev/null +++ b/f-assistant/src/rag/rag.controller.ts @@ -0,0 +1,13 @@ +import { Body, Controller, Post } from '@nestjs/common'; +import { RagService } from './rag.service.js'; +import { AnswerPack } from './schemas.js'; + +@Controller('answer') +export class RagController { + constructor(private readonly ragService: RagService) {} + + @Post() + async answer(@Body('query') query: string): Promise { + return this.ragService.answer(query); + } +} diff --git a/f-assistant/src/rag/rag.module.ts b/f-assistant/src/rag/rag.module.ts new file mode 100644 index 00000000..c8e9b85c --- /dev/null +++ b/f-assistant/src/rag/rag.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +import { ChunkingService } from './chunking.service.js'; +import { RetrieverService } from './retriever.service.js'; +import { GeneratorService } from './generator.service.js'; +import { PromptService } from './prompt.service.js'; +import { RagService } from './rag.service.js'; +import { RagController } from './rag.controller.js'; +import { StorageModule } from '../storage/storage.module.js'; +import { ConfigModule } from '../config/config.module.js'; + +@Module({ + imports: [StorageModule, ConfigModule], + providers: [ChunkingService, RetrieverService, GeneratorService, PromptService, RagService], + exports: [ChunkingService, RetrieverService, GeneratorService, PromptService, RagService], + controllers: [RagController], +}) +export class RagModule {} +export { RagService } from './rag.service.js'; diff --git a/f-assistant/src/rag/rag.service.ts b/f-assistant/src/rag/rag.service.ts new file mode 100644 index 00000000..4e1b5e87 --- /dev/null +++ b/f-assistant/src/rag/rag.service.ts @@ -0,0 +1,42 @@ +import { Injectable } from '@nestjs/common'; +import { RetrieverService } from './retriever.service.js'; +import { GeneratorService } from './generator.service.js'; +import { answerPackSchema, AnswerPack } from './schemas.js'; + +@Injectable() +export class RagService { + constructor(private readonly retriever: RetrieverService, private readonly generator: GeneratorService) {} + + async answer(query: string): Promise { + const context = await this.retrieve(query); + const pack = await this.generator.generate({ + source: { + type: 'website', + repo: 'Foblex/f-flow', + url: 'https://foblex.dev/questions', + }, + context, + query, + }); + return answerPackSchema.parse(pack); + } + + async answerFromUrl(url: string): Promise { + const context = await this.retrieve(url); + const pack = await this.generator.generate({ + source: { + type: url.includes('/discussions/') ? 'github_discussion' : 'github_issue', + repo: 'Foblex/f-flow', + url, + }, + context, + query: url, + }); + return answerPackSchema.parse(pack); + } + + private async retrieve(query: string): Promise { + const results = await this.retriever.keywordSearch(query); + return results.map((r) => `${r.path} ${r.lines ?? ''}: ${r.content.slice(0, 400)}`); + } +} diff --git a/f-assistant/src/rag/retriever.service.ts b/f-assistant/src/rag/retriever.service.ts new file mode 100644 index 00000000..2f8681e0 --- /dev/null +++ b/f-assistant/src/rag/retriever.service.ts @@ -0,0 +1,45 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { QdrantClient } from '@qdrant/js-client-rest'; +import { ConfigService } from '@nestjs/config'; +import { Chunk } from './chunking.service.js'; +import { SourceChunk } from '../storage/types.js'; + +@Injectable() +export class RetrieverService { + private readonly logger = new Logger(RetrieverService.name); + private readonly client: QdrantClient; + private readonly collection: string; + + constructor(configService: ConfigService) { + const url = configService.get('QDRANT_URL'); + this.collection = configService.get('QDRANT_COLLECTION') ?? 'foblex_flow'; + this.client = new QdrantClient({ url }); + } + + async upsertChunks(chunks: Chunk[]) { + // store minimal payload; embeddings assumed precomputed upstream + await this.client.upsert(this.collection, { + points: chunks.map((chunk, idx) => ({ + id: `${chunk.id}-${idx}`, + vector: [0], + payload: chunk, + })), + }); + } + + async keywordSearch(query: string): Promise { + try { + const res = await this.client.scroll(this.collection, { + with_payload: true, + limit: 20, + }); + const items = (res?.result ?? []) as any[]; + return items + .map((item) => item.payload as any as SourceChunk) + .filter((p) => p.content?.toLowerCase().includes(query.toLowerCase())); + } catch (err) { + this.logger.warn(`Falling back to empty search: ${err}`); + return []; + } + } +} diff --git a/f-assistant/src/rag/schemas.ts b/f-assistant/src/rag/schemas.ts new file mode 100644 index 00000000..e7c5dd70 --- /dev/null +++ b/f-assistant/src/rag/schemas.ts @@ -0,0 +1,34 @@ +import { z } from 'zod'; + +export const citationSchema = z.object({ + kind: z.string(), + path: z.string().optional(), + ref: z.string().optional(), + lines: z.string().optional(), + url: z.string().optional(), +}); + +export const draftSchema = z.object({ + id: z.enum(['A', 'B', 'C']), + tone: z.string(), + description: z.string(), + text: z.string(), + citations: z.array(citationSchema), +}); + +export const answerPackSchema = z.object({ + source: z.object({ + type: z.enum(['github_issue', 'github_discussion', 'website']), + repo: z.literal('Foblex/f-flow'), + url: z.string(), + number: z.number().optional(), + author: z.string().optional(), + title: z.string().optional(), + }), + drafts: z.array(draftSchema).length(3), + confidence: z.number().min(0).max(1), + missing_context: z.array(z.string()), +}); + +export type AnswerPack = z.infer; +export type Draft = z.infer; diff --git a/f-assistant/src/scripts/ingest-github.ts b/f-assistant/src/scripts/ingest-github.ts new file mode 100644 index 00000000..d85c35ea --- /dev/null +++ b/f-assistant/src/scripts/ingest-github.ts @@ -0,0 +1,19 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { IngestModule } from '../ingest/ingest.module.js'; +import { GithubIngestService } from '../ingest/github-ingest.service.js'; +import yargs from 'yargs/yargs'; +import { hideBin } from 'yargs/helpers'; +import * as dotenv from 'dotenv'; + +dotenv.config(); + +async function bootstrap() { + const argv = await yargs(hideBin(process.argv)).option('since', { type: 'string' }).parse(); + const app = await NestFactory.createApplicationContext(IngestModule); + const service = app.get(GithubIngestService); + await service.ingestIssues(argv.since as string | undefined); + await app.close(); +} + +bootstrap(); diff --git a/f-assistant/src/scripts/ingest-repo.ts b/f-assistant/src/scripts/ingest-repo.ts new file mode 100644 index 00000000..525b11bd --- /dev/null +++ b/f-assistant/src/scripts/ingest-repo.ts @@ -0,0 +1,17 @@ +import 'reflect-metadata'; +import { NestFactory } from '@nestjs/core'; +import { IngestModule } from '../ingest/ingest.module.js'; +import { RepoIngestService } from '../ingest/repo-ingest.service.js'; +import { ConfigModule } from '../config/config.module.js'; +import * as dotenv from 'dotenv'; + +dotenv.config(); + +async function bootstrap() { + const app = await NestFactory.createApplicationContext(IngestModule, { imports: [ConfigModule] }); + const service = app.get(RepoIngestService); + await service.ingest('..'); + await app.close(); +} + +bootstrap(); diff --git a/f-assistant/src/storage/drafts.service.ts b/f-assistant/src/storage/drafts.service.ts new file mode 100644 index 00000000..3d6711d8 --- /dev/null +++ b/f-assistant/src/storage/drafts.service.ts @@ -0,0 +1,28 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { AnswerPack } from '../rag/schemas.js'; + +@Injectable() +export class DraftsService { + private readonly prisma = new PrismaClient(); + + async savePack(pack: AnswerPack) { + await Promise.all( + pack.drafts.map((draft) => + this.prisma.draft.create({ + data: { + sourceType: pack.source.type, + sourceUrl: pack.source.url, + sourceNumber: pack.source.number ?? 0, + author: pack.source.author ?? 'unknown', + title: pack.source.title ?? '', + draftId: draft.id, + tone: draft.tone, + text: draft.text, + citations: JSON.stringify(draft.citations), + }, + }), + ), + ); + } +} diff --git a/f-assistant/src/storage/jobs.service.ts b/f-assistant/src/storage/jobs.service.ts new file mode 100644 index 00000000..304fed63 --- /dev/null +++ b/f-assistant/src/storage/jobs.service.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; + +@Injectable() +export class JobsService { + private readonly prisma = new PrismaClient(); + + async enqueue(sourceUrl: string, payload: any) { + return this.prisma.job.create({ data: { status: 'queued', sourceUrl, payload: JSON.stringify(payload) } }); + } +} diff --git a/f-assistant/src/storage/storage.module.ts b/f-assistant/src/storage/storage.module.ts new file mode 100644 index 00000000..11c9945e --- /dev/null +++ b/f-assistant/src/storage/storage.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { DraftsService } from './drafts.service.js'; +import { JobsService } from './jobs.service.js'; + +@Module({ + providers: [DraftsService, JobsService], + exports: [DraftsService, JobsService], +}) +export class StorageModule {} diff --git a/f-assistant/src/storage/types.ts b/f-assistant/src/storage/types.ts new file mode 100644 index 00000000..f3a4712e --- /dev/null +++ b/f-assistant/src/storage/types.ts @@ -0,0 +1,10 @@ +export interface SourceChunk { + id: string; + source: string; + kind: string; + path: string; + ref?: string; + lines?: string; + content: string; + embedding?: string; +} diff --git a/f-assistant/src/webhook/github-webhook.controller.ts b/f-assistant/src/webhook/github-webhook.controller.ts new file mode 100644 index 00000000..d4c54e77 --- /dev/null +++ b/f-assistant/src/webhook/github-webhook.controller.ts @@ -0,0 +1,19 @@ +import { Body, Controller, Headers, HttpCode, Post } from '@nestjs/common'; +import { GithubWebhookService } from './github-webhook.service.js'; +import { SignatureService } from './signature.service.js'; + +@Controller('/webhook/github') +export class GithubWebhookController { + constructor( + private readonly webhookService: GithubWebhookService, + private readonly signatureService: SignatureService, + ) {} + + @Post() + @HttpCode(200) + async handle(@Body() payload: any, @Headers('x-hub-signature-256') signature?: string) { + this.signatureService.verifySignature(signature, JSON.stringify(payload)); + await this.webhookService.handleEvent(payload); + return { ok: true }; + } +} diff --git a/f-assistant/src/webhook/github-webhook.service.ts b/f-assistant/src/webhook/github-webhook.service.ts new file mode 100644 index 00000000..4a5a4a2a --- /dev/null +++ b/f-assistant/src/webhook/github-webhook.service.ts @@ -0,0 +1,42 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { RagService } from '../rag/rag.module.js'; +import { GithubDiscussionsService } from '../integrations/github/github-discussions.service.js'; +import { TelegramService } from '../integrations/telegram/telegram.service.js'; +import { DraftsService } from '../storage/drafts.service.js'; + +@Injectable() +export class GithubWebhookService { + private readonly logger = new Logger(GithubWebhookService.name); + + constructor( + private readonly ragService: RagService, + private readonly discussionsService: GithubDiscussionsService, + private readonly telegramService: TelegramService, + private readonly draftsService: DraftsService, + ) {} + + async handleEvent(payload: any) { + const { action, issue, discussion } = payload; + if (!action) return; + if (issue && ['opened'].includes(action)) { + await this.createDraftFromIssue(issue.html_url); + } + if (discussion && ['created'].includes(action)) { + await this.createDraftFromDiscussion(discussion.html_url); + } + } + + private async createDraftFromIssue(url: string) { + const pack = await this.ragService.answerFromUrl(url); + await this.telegramService.sendDraftPack(pack); + await this.draftsService.savePack(pack); + this.logger.log(`Draft pack created for issue ${url}`); + } + + private async createDraftFromDiscussion(url: string) { + const pack = await this.ragService.answerFromUrl(url); + await this.telegramService.sendDraftPack(pack); + await this.draftsService.savePack(pack); + this.logger.log(`Draft pack created for discussion ${url}`); + } +} diff --git a/f-assistant/src/webhook/signature.service.ts b/f-assistant/src/webhook/signature.service.ts new file mode 100644 index 00000000..6dd90e27 --- /dev/null +++ b/f-assistant/src/webhook/signature.service.ts @@ -0,0 +1,23 @@ +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import crypto from 'crypto'; +import { ConfigService } from '@nestjs/config'; + +@Injectable() +export class SignatureService { + constructor(private readonly configService: ConfigService) {} + + verifySignature(signature: string | undefined, rawBody: string) { + const secret = this.configService.get('GITHUB_WEBHOOK_SECRET'); + if (!secret) { + return; + } + if (!signature) { + throw new UnauthorizedException('Missing signature'); + } + const hmac = crypto.createHmac('sha256', secret); + const digest = 'sha256=' + hmac.update(rawBody).digest('hex'); + if (!crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(digest))) { + throw new UnauthorizedException('Invalid signature'); + } + } +} diff --git a/f-assistant/src/webhook/webhook.module.ts b/f-assistant/src/webhook/webhook.module.ts new file mode 100644 index 00000000..8f6aa2e1 --- /dev/null +++ b/f-assistant/src/webhook/webhook.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { GithubWebhookController } from './github-webhook.controller.js'; +import { GithubWebhookService } from './github-webhook.service.js'; +import { SignatureService } from './signature.service.js'; +import { RagModule } from '../rag/rag.module.js'; +import { IntegrationsGithubModule } from '../integrations/github/github.module.js'; +import { TelegramModule } from '../integrations/telegram/telegram.module.js'; + +@Module({ + imports: [RagModule, IntegrationsGithubModule, TelegramModule], + controllers: [GithubWebhookController], + providers: [GithubWebhookService, SignatureService], +}) +export class WebhookModule {} diff --git a/f-assistant/tsconfig.json b/f-assistant/tsconfig.json new file mode 100644 index 00000000..05b83cff --- /dev/null +++ b/f-assistant/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "module": "CommonJS", + "target": "ES2022", + "moduleResolution": "node", + "outDir": "dist", + "esModuleInterop": true, + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "sourceMap": true, + "strict": true, + "baseUrl": "./", + "paths": { + "@/*": ["src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +}