Skip to content

Commit 6a3761f

Browse files
vishnoianilvishnoianil
authored
Add API's for document add/replace/list/get/remove (#794)
Signed-off-by: Anil Vishnoi <vishnoianil@gmail.com>
1 parent cefdcba commit 6a3761f

File tree

8 files changed

+335
-5
lines changed

8 files changed

+335
-5
lines changed

‎src/app/api/documents/add/route.ts‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// src/app/api/documents/add/route.ts
2+
3+
'use server';
4+
import { NextRequest, NextResponse } from 'next/server';
5+
import * as git from 'isomorphic-git';
6+
import fs from 'fs';
7+
import path from 'path';
8+
import { cloneTaxonomyDocsRepo, DOC_POOL_DIR, TAXONOMY_DOCS_ROOT_DIR } from '@/app/api/utils';
9+
import { devLog } from '@/utils/devlog';
10+
11+
/**
12+
* Handler to upload new files and replace existing files to the document pool of taxonomy knowledge doc repo.
13+
*/
14+
export async function POST(req: NextRequest) {
15+
try {
16+
const body = await req.json();
17+
const { newFiles } = body;
18+
const docsRepoPath = await cloneTaxonomyDocsRepo();
19+
20+
// If the repository was not cloned, return an error
21+
if (!docsRepoPath) {
22+
return NextResponse.json({ error: 'Failed to clone taxonomy knowledge docs repository' }, { status: 500 });
23+
}
24+
25+
// Checkout the main branch
26+
await git.checkout({ fs, dir: docsRepoPath, ref: 'main' });
27+
28+
const subDirectory = DOC_POOL_DIR;
29+
30+
const newDocsDirPath = path.join(docsRepoPath, subDirectory);
31+
32+
if (!fs.existsSync(newDocsDirPath)) {
33+
fs.mkdirSync(newDocsDirPath, { recursive: true });
34+
devLog(`New sub directory ${newDocsDirPath} created successfully.`);
35+
}
36+
37+
// Write the files to the repository
38+
for (const file of newFiles) {
39+
const filePath = path.join(newDocsDirPath, file.fileName);
40+
devLog(`Writing file to ${filePath} in taxonomy knowledge docs repository.`);
41+
fs.writeFileSync(filePath, file.fileContent);
42+
}
43+
44+
const filenames = newFiles.map((file: { fileName: string; fileContent: string }) => path.join(subDirectory, file.fileName));
45+
46+
// Stage the files
47+
await git.add({ fs, dir: docsRepoPath, filepath: '.' });
48+
await git.remove({ fs, dir: docsRepoPath, filepath: '.' });
49+
50+
// Commit the files
51+
await git.commit({
52+
fs,
53+
dir: docsRepoPath,
54+
author: { name: 'instructlab-ui', email: 'ui@instructlab.ai' },
55+
message: `File uploaded: ${filenames}\n\nSigned-off-by: ui@instructlab.ai`
56+
});
57+
58+
devLog(`Successfully uploaded following file to taxonomy knowledge docs repository: ${filenames}`);
59+
60+
const origTaxonomyDocsRepoDir = path.join(TAXONOMY_DOCS_ROOT_DIR, '/taxonomy-knowledge-docs');
61+
return NextResponse.json(
62+
{
63+
repoUrl: origTaxonomyDocsRepoDir,
64+
documentNames: filenames
65+
},
66+
{ status: 201 }
67+
);
68+
} catch (error) {
69+
console.error('Failed to upload knowledge file:', error);
70+
return NextResponse.json({ error: 'Failed to upload knowledge file' }, { status: 500 });
71+
}
72+
}

‎src/app/api/documents/get/route.ts‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// src/app/api/documents/get.ts
2+
3+
'use server';
4+
import { NextRequest, NextResponse } from 'next/server';
5+
import * as git from 'isomorphic-git';
6+
import fs from 'fs';
7+
import path from 'path';
8+
import { DOC_POOL_DIR, findTaxonomyDocRepoPath } from '@/app/api/utils';
9+
import { KnowledgeFile } from '@/types';
10+
11+
const BASE_BRANCH = 'main';
12+
13+
/**
14+
* Function to retrieve knowledge file content from a document pool.
15+
* @param filename - Name of the file to retrieve
16+
* @returns A KnowledgeFile object with the content
17+
*/
18+
const getKnowledgeFiles = async (filename: string): Promise<KnowledgeFile> => {
19+
const REPO_DIR = findTaxonomyDocRepoPath();
20+
21+
// Ensure the repository path exists
22+
if (!fs.existsSync(REPO_DIR)) {
23+
throw new Error('Taxonomy knowledge doc repository does not exist. No files present.');
24+
}
25+
26+
// Check if the branch exists
27+
const branches = await git.listBranches({ fs, dir: REPO_DIR });
28+
if (!branches.includes(BASE_BRANCH)) {
29+
throw new Error(`Branch "${BASE_BRANCH}" does not exist.`);
30+
}
31+
32+
// Checkout the specified branch
33+
await git.checkout({ fs, dir: REPO_DIR, ref: BASE_BRANCH });
34+
35+
// check if the file exists in the document pool
36+
const filePath = path.join(REPO_DIR, DOC_POOL_DIR, filename);
37+
38+
if (!fs.existsSync(filePath)) {
39+
throw new Error(`File "${filename}" does not exist in document pool.`);
40+
}
41+
42+
try {
43+
// Retrieve the latest commit SHA for the file on the specified branch
44+
const logs = await git.log({
45+
fs,
46+
dir: REPO_DIR,
47+
ref: BASE_BRANCH,
48+
filepath: path.join(DOC_POOL_DIR, filename),
49+
depth: 1 // Only the latest commit
50+
});
51+
52+
if (logs.length === 0) {
53+
throw new Error(`File "${filename}" exist, but has no related commit history.`);
54+
}
55+
56+
const latestCommit = logs[0];
57+
58+
const commitDate = new Date(latestCommit.commit.committer.timestamp * 1000).toISOString();
59+
60+
// Read the file content
61+
const fileContent = fs.readFileSync(filePath, 'utf-8');
62+
63+
const knowledgeFile: KnowledgeFile = {
64+
filename: path.basename(filename),
65+
content: fileContent,
66+
commitDate: commitDate
67+
};
68+
return knowledgeFile;
69+
} catch (error) {
70+
console.error(`Failed to read file ${filename}:`, error);
71+
throw new Error(`File "${filename}" does not exist in document pool.`);
72+
}
73+
};
74+
75+
/**
76+
* Handler to retrieve knowledge file content
77+
*/
78+
export async function GET(request: NextRequest) {
79+
const url = new URL(request.url);
80+
const filename = url.searchParams.get('filename');
81+
if (filename != null) {
82+
try {
83+
const knowledgeFile = await getKnowledgeFiles(filename);
84+
return NextResponse.json({ file: knowledgeFile }, { status: 200 });
85+
} catch (error) {
86+
console.error(`Failed to retrieve content of the file: ${filename}`, error);
87+
return NextResponse.json({ error: (error as Error).message }, { status: 500 });
88+
}
89+
} else {
90+
console.error(`File name must be not empty.`);
91+
return NextResponse.json({ error: `File name must be not empty.` }, { status: 500 });
92+
}
93+
}

‎src/app/api/documents/list/route.ts‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
// src/app/api/documents/list/route.ts
2+
3+
'use server';
4+
import { NextResponse } from 'next/server';
5+
import * as git from 'isomorphic-git';
6+
import fs from 'fs';
7+
import path from 'path';
8+
import { DOC_POOL_DIR, findTaxonomyDocRepoPath } from '@/app/api/utils';
9+
import { KnowledgeFile } from '@/types';
10+
import { devLog } from '@/utils/devlog';
11+
12+
const BASE_BRANCH = 'main';
13+
14+
/**
15+
* Function to retrieve list of documents from taxonomy-knowledge-docs document pool.
16+
* @returns An array of document name.
17+
*/
18+
const getKnowledgeFiles = async (): Promise<KnowledgeFile[]> => {
19+
const REPO_DIR = findTaxonomyDocRepoPath();
20+
const knowledgeFiles: KnowledgeFile[] = [];
21+
22+
// Ensure the repository path exists
23+
if (!fs.existsSync(REPO_DIR)) {
24+
devLog("Taxonomy knowledge doc directory doesn't exist at :", REPO_DIR);
25+
return knowledgeFiles;
26+
}
27+
28+
// Check if the branch exists
29+
const branches = await git.listBranches({ fs, dir: REPO_DIR });
30+
if (!branches.includes(BASE_BRANCH)) {
31+
throw new Error(`Branch "${BASE_BRANCH}" does not exist.`);
32+
}
33+
34+
// Checkout the specified branch
35+
await git.checkout({ fs, dir: REPO_DIR, ref: BASE_BRANCH });
36+
37+
// Read all files in the repository root directory
38+
39+
const docPoolDir = path.join(REPO_DIR, DOC_POOL_DIR);
40+
41+
// Ensure the doc-pool directory exist
42+
if (!fs.existsSync(docPoolDir)) {
43+
devLog(`${DOC_POOL_DIR} directory doesn't exist.`);
44+
return knowledgeFiles;
45+
}
46+
47+
const allFiles = await fs.promises.readdir(docPoolDir, { recursive: true });
48+
49+
// Filter for Markdown files only
50+
const markdownFiles = allFiles.filter((file) => path.extname(file).toLowerCase() === '.md');
51+
52+
for (const file of markdownFiles) {
53+
const filePath = path.join(docPoolDir, file);
54+
55+
// Check if the file is a regular file
56+
const stat = fs.statSync(filePath);
57+
if (!stat.isFile()) {
58+
continue;
59+
}
60+
61+
try {
62+
// Retrieve the latest commit SHA for the file on the specified branch
63+
const logs = await git.log({
64+
fs,
65+
dir: REPO_DIR,
66+
ref: BASE_BRANCH,
67+
filepath: path.join(DOC_POOL_DIR, file),
68+
depth: 1 // Only the latest commit
69+
});
70+
71+
if (logs.length === 0) {
72+
// No commits found for this file; skip it
73+
continue;
74+
}
75+
76+
const latestCommit = logs[0];
77+
78+
const commitDate = new Date(latestCommit.commit.committer.timestamp * 1000).toISOString();
79+
knowledgeFiles.push({
80+
filename: path.basename(file),
81+
commitDate: commitDate
82+
});
83+
} catch (error) {
84+
console.error(`Failed to retrieve commit for file ${file}:`, error);
85+
throw new Error(`Failed to retrieve commit for file: ${error}`);
86+
}
87+
}
88+
return knowledgeFiles;
89+
};
90+
91+
/**
92+
* Handler to retrieve list of knowledge file from the taxonomy knowledge doc document pool.
93+
*/
94+
export async function GET() {
95+
try {
96+
const knowledgeFiles = await getKnowledgeFiles();
97+
return NextResponse.json({ files: knowledgeFiles }, { status: 200 });
98+
} catch (error) {
99+
console.error('Failed to fetch list of files from document pool:', error);
100+
return NextResponse.json({ error: (error as Error).message }, { status: 500 });
101+
}
102+
}

‎src/app/api/documents/remove/route.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// src/app/api/documents/remove/route.ts
2+
3+
'use server';
4+
import { NextRequest, NextResponse } from 'next/server';
5+
import * as git from 'isomorphic-git';
6+
import fs from 'fs';
7+
import path from 'path';
8+
import { DOC_POOL_DIR, findTaxonomyDocRepoPath } from '@/app/api/utils';
9+
import { devLog } from '@/utils/devlog';
10+
11+
/**
12+
* Handler to delete knowledge file from taxonomy knowledge doc repo's document pool
13+
*/
14+
export async function POST(req: NextRequest) {
15+
try {
16+
const body = await req.json();
17+
const { fileName } = body;
18+
const docsRepoPath = findTaxonomyDocRepoPath();
19+
20+
// If the repository was not cloned, return an error
21+
if (!docsRepoPath && docsRepoPath == '') {
22+
console.error('Taxonomy knowledge docs repository is not present on the host');
23+
return NextResponse.json(
24+
{ error: 'Failed to delete the document from document pool. Taxonomy knowledge docs repository is not present.' },
25+
{ status: 500 }
26+
);
27+
}
28+
29+
// Checkout the main branch
30+
await git.checkout({ fs, dir: docsRepoPath, ref: 'main' });
31+
32+
const newDocsDirPath = path.join(docsRepoPath, DOC_POOL_DIR);
33+
34+
if (!fs.existsSync(newDocsDirPath)) {
35+
console.error(`Document pool directory doesn't exist: ${docsRepoPath}`);
36+
return NextResponse.json({ error: 'Failed to delete the file from document pool. File does not exists.' }, { status: 500 });
37+
}
38+
39+
const filePath = path.join(newDocsDirPath, fileName);
40+
devLog(`Deleting file ${filePath} from document pool.`);
41+
fs.rmSync(filePath);
42+
43+
// Stage the files
44+
await git.add({ fs, dir: docsRepoPath, filepath: '.' });
45+
await git.remove({ fs, dir: docsRepoPath, filepath: '.' });
46+
47+
// Commit the files
48+
await git.commit({
49+
fs,
50+
dir: docsRepoPath,
51+
author: { name: 'instructlab-ui', email: 'ui@instructlab.ai' },
52+
message: `File deleted from document pool: ${fileName}\n\nSigned-off-by: ui@instructlab.ai`
53+
});
54+
55+
devLog(`Successfully deleted file ${fileName} from document pool.`);
56+
57+
return NextResponse.json({ message: `Successfully deleted file ${fileName} from document pool.` }, { status: 201 });
58+
} catch (error) {
59+
console.error('Failed to upload knowledge files:', error);
60+
return NextResponse.json({ error: 'Failed to upload knowledge files' }, { status: 500 });
61+
}
62+
}

‎src/app/api/utils.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import fs from 'fs';
44
import * as git from 'isomorphic-git';
55
import http from 'isomorphic-git/http/node';
66

7+
export const DOC_POOL_DIR = 'doc-pool';
78
export const TAXONOMY_DOCS_ROOT_DIR = process.env.NEXT_PUBLIC_TAXONOMY_ROOT_DIR || '';
89
export const TAXONOMY_DOCS_CONTAINER_MOUNT_DIR = '/tmp/.instructlab-ui';
910
export const TAXONOMY_KNOWLEDGE_DOCS_REPO_URL =
@@ -24,7 +25,7 @@ export const cloneTaxonomyDocsRepo = async (): Promise<string | null> => {
2425
return null;
2526
}
2627

27-
const taxonomyDocsDirectoryPath = path.join(remoteTaxonomyRepoDirFinal, '/taxonomy-knowledge-docs');
28+
const taxonomyDocsDirectoryPath = path.join(path.dirname(remoteTaxonomyRepoDirFinal), '/taxonomy-knowledge-docs');
2829

2930
if (fs.existsSync(taxonomyDocsDirectoryPath)) {
3031
console.log(`Using existing taxonomy knowledge docs repository at ${TAXONOMY_DOCS_ROOT_DIR}/taxonomy-knowledge-docs.`);

‎src/components/Contribute/Knowledge/KnowledgeWizard/UploadFile.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,7 @@ export const UploadFile: React.FunctionComponent<UploadFileProps> = ({ existingF
302302
<FlexItem>
303303
<MultipleFileUploadStatus statusToggleText={`Existing uploaded files`} statusToggleIcon={statusIcon} aria-label="Existing uploads">
304304
{existingFiles.map((file: KnowledgeFile) => {
305-
const fileObject = new File([file.content], file.filename, { type: 'text/plain' });
305+
const fileObject = new File([file.content ? file.content : ''], file.filename, { type: 'text/plain' });
306306
return (
307307
<MultipleFileUploadStatusItem
308308
file={fileObject}

‎src/components/Contribute/Utils/documentUtils.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ export const UploadKnowledgeDocuments = async (
3434
)
3535
);
3636

37-
knowledgeFormData.uploadedFiles.map((file: { filename: string; content: string }) => {
38-
updatedExistingFiles.push({ fileName: file.filename, fileContent: file.content });
37+
knowledgeFormData.uploadedFiles.map((file: { filename: string; content?: string }) => {
38+
updatedExistingFiles.push({ fileName: file.filename, fileContent: file.content ? file.content : '' });
3939
});
4040
// Trigger the upload only if all the newly uploaded files were read successfully and there are existing uploaded files.
4141
if (newFiles.length === knowledgeFormData.filesToUpload.length && (newFiles.length !== 0 || updatedExistingFiles.length !== 0)) {

‎src/types/index.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ export interface SkillSeedExample extends SeedExample {
142142

143143
export interface KnowledgeFile {
144144
filename: string;
145-
content: string;
145+
content?: string;
146146
commitSha?: string;
147147
commitDate?: string;
148148
}

0 commit comments

Comments
 (0)