Skip to content

Commit b15956a

Browse files
yangbobo2021yangbobo2021
authored
Merge pull request #591 from devchat-ai/add_log_in_codecompletion
feat: Add logging in code completion
2 parents 46e17b9 + adec472 commit b15956a

File tree

1 file changed

+129
-110
lines changed

1 file changed

+129
-110
lines changed

‎src/contributes/codecomplete/promptCreator.ts‎

Lines changed: 129 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -564,138 +564,157 @@ export async function createTaskDescriptionContext() {
564564
}
565565

566566
export async function createPrompt(filePath: string, fileContent: string, line: number, column: number, posoffset: number, recentEdits: RecentEdit[]) {
567-
const commentPrefix = await getCommentPrefix(filePath);
567+
try {
568+
const commentPrefix = await getCommentPrefix(filePath);
568569

569-
let { prefix, suffix } = await currentFileContext(filePath, fileContent, line, column);
570-
571-
let tokenCount = countTokens(prefix) + countTokens(suffix);
572-
573-
let taskDescriptionContextWithCommentPrefix = "";
574-
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
575-
const taskDescriptionContext = await createTaskDescriptionContext();
576-
if (taskDescriptionContext) {
577-
taskDescriptionContext.split("\n").forEach(line => {
578-
taskDescriptionContextWithCommentPrefix += `${commentPrefix}<filename>task: ${line}\n`;
579-
});
570+
let { prefix, suffix } = await currentFileContext(filePath, fileContent, line, column);
580571

581-
taskDescriptionContextWithCommentPrefix += "\n\n\n\n";
582-
}
572+
let tokenCount = countTokens(prefix) + countTokens(suffix);
583573

584-
const taskDescriptionContextToken = countTokens(taskDescriptionContextWithCommentPrefix);
585-
if (tokenCount + taskDescriptionContextToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
586-
tokenCount += taskDescriptionContextToken;
587-
} else {
588-
taskDescriptionContextWithCommentPrefix = "";
589-
}
590-
}
574+
let taskDescriptionContextWithCommentPrefix = "";
575+
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
576+
try {
577+
const taskDescriptionContext = await createTaskDescriptionContext();
578+
if (taskDescriptionContext) {
579+
taskDescriptionContext.split("\n").forEach(line => {
580+
taskDescriptionContextWithCommentPrefix += `${commentPrefix}<filename>task: ${line}\n`;
581+
});
591582

592-
// let gitDiffContext = GitDiffWatcher.getInstance().getGitDiffResult();
593-
// if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000) && gitDiffContext.length > 0) {
594-
// const gitDiffContextToken = countTokens(gitDiffContext);
595-
// if (tokenCount + gitDiffContextToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
596-
// tokenCount += gitDiffContextToken;
597-
// gitDiffContext = "<git_diff_start>" + gitDiffContext + "<git_diff_end>\n\n\n\n";
598-
// } else {
599-
// gitDiffContext = "";
600-
// }
601-
// }
602-
let gitDiffContext = "";
603-
604-
let callDefContext = "";
605-
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
606-
const callCodeBlocks = await createContextCallDefine(filePath, fileContent, posoffset);
607-
for (const callCodeBlock of callCodeBlocks) {
608-
const callBlockToken = countTokens(callCodeBlock.codeblock);
609-
if (tokenCount + callBlockToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
610-
break;
611-
}
583+
taskDescriptionContextWithCommentPrefix += "\n\n\n\n";
584+
}
612585

613-
tokenCount += callBlockToken;
614-
callDefContext += `${commentPrefix}<filename>call function define:\n\n ${callCodeBlock.filepath}\n\n`;
615-
callDefContext += `${callCodeBlock.codeblock}\n\n\n\n`;
586+
const taskDescriptionContextToken = countTokens(taskDescriptionContextWithCommentPrefix);
587+
if (tokenCount + taskDescriptionContextToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
588+
tokenCount += taskDescriptionContextToken;
589+
} else {
590+
taskDescriptionContextWithCommentPrefix = "";
591+
}
592+
} catch (error) {
593+
logger.channel()?.error("Error creating task description context:", error);
594+
}
616595
}
617-
}
618596

619-
let similarBlockContext = "";
620-
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
621-
let similarTokens = 0;
622-
const similarContexts: {file: string, text: string}[] = await findSimilarCodeBlock(filePath, fileContent, line, column);
597+
let gitDiffContext = "";
623598

624-
for (const similarContext of similarContexts) {
625-
const blockToken = countTokens(similarContext.text);
626-
if (tokenCount + blockToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
627-
continue;
599+
let callDefContext = "";
600+
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
601+
try {
602+
const callCodeBlocks = await createContextCallDefine(filePath, fileContent, posoffset);
603+
for (const callCodeBlock of callCodeBlocks) {
604+
const callBlockToken = countTokens(callCodeBlock.codeblock);
605+
if (tokenCount + callBlockToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
606+
break;
607+
}
608+
609+
tokenCount += callBlockToken;
610+
callDefContext += `${commentPrefix}<filename>call function define:\n\n ${callCodeBlock.filepath}\n\n`;
611+
callDefContext += `${callCodeBlock.codeblock}\n\n\n\n`;
612+
}
613+
} catch (error) {
614+
logger.channel()?.error("Error creating call definition context:", error);
628615
}
629-
similarTokens += blockToken;
630-
if (similarTokens > CONTEXT_SIMILAR_LIMITED_SIZE) {
631-
continue;
616+
}
617+
618+
let similarBlockContext = "";
619+
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
620+
try {
621+
let similarTokens = 0;
622+
const similarContexts: {file: string, text: string}[] = await findSimilarCodeBlock(filePath, fileContent, line, column);
623+
624+
for (const similarContext of similarContexts) {
625+
const blockToken = countTokens(similarContext.text);
626+
if (tokenCount + blockToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
627+
continue;
628+
}
629+
similarTokens += blockToken;
630+
if (similarTokens > CONTEXT_SIMILAR_LIMITED_SIZE) {
631+
continue;
632+
}
633+
634+
tokenCount += blockToken;
635+
similarBlockContext += `${commentPrefix}<filename>similar blocks:\n\n ${similarContext.file}\n\n`;
636+
similarBlockContext += `${similarContext.text}\n\n\n\n`;
637+
}
638+
} catch (error) {
639+
logger.channel()?.error("Error creating similar block context:", error);
632640
}
641+
}
642+
643+
let symbolContext = "";
644+
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
645+
try {
646+
const symbolDefines: { filepath: string, node: Parser.SyntaxNode, codeblock: string }[] = await symbolDefinesContext(filePath, fileContent, line, column);
647+
for (const symbolDefine of symbolDefines) {
648+
const countSymboleToken = countTokens(symbolDefine.codeblock);
649+
if (tokenCount + countSymboleToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
650+
break;
651+
}
633652

634-
tokenCount += blockToken;
635-
similarBlockContext += `${commentPrefix}<filename>similar blocks:\n\n ${similarContext.file}\n\n`;
636-
similarBlockContext += `${similarContext.text}\n\n\n\n`;
653+
tokenCount += countSymboleToken;
654+
symbolContext += `${commentPrefix}<filename>symbol defines:\n\n ${symbolDefine.filepath}\n\n`;
655+
symbolContext += `${commentPrefix}this is type of variable: ${symbolDefine.node.text}\n\n`;
656+
symbolContext += `${symbolDefine.codeblock}\n\n\n\n`;
657+
}
658+
} catch (error) {
659+
logger.channel()?.error("Error creating symbol context:", error);
660+
}
637661
}
638-
}
639662

640-
let symbolContext = "";
641-
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
642-
const symbolDefines: { filepath: string, node: Parser.SyntaxNode, codeblock: string }[] = await symbolDefinesContext(filePath, fileContent, line, column);
643-
for (const symbolDefine of symbolDefines ) {
644-
const countSymboleToken = countTokens(symbolDefine.codeblock);
645-
if (tokenCount + countSymboleToken > DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
646-
break;
663+
let recentEditContext = "";
664+
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
665+
try {
666+
recentEditContext = await createRecentEditContext(recentEdits, filePath);
667+
668+
const countRecentToken = countTokens(recentEditContext);
669+
if (tokenCount + countRecentToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
670+
tokenCount += countRecentToken;
671+
} else {
672+
recentEditContext = "";
673+
}
674+
} catch (error) {
675+
logger.channel()?.error("Error creating recent edit context:", error);
647676
}
677+
}
648678

649-
tokenCount += countSymboleToken;
650-
symbolContext += `${commentPrefix}<filename>symbol defines:\n\n ${symbolDefine.filepath}\n\n`;
651-
symbolContext += `${commentPrefix}this is type of variable: ${symbolDefine.node.text}\n\n`;
652-
symbolContext += `${symbolDefine.codeblock}\n\n\n\n`;
679+
let neighborFileContext = "";
680+
if (tokenCount < 200) {
681+
try {
682+
const neighborFiles = await findNeighborFileContext(filePath, fileContent, line, column);
683+
if (neighborFiles.length > 0) {
684+
const countFileToken = countTokens(neighborFiles[0].text);
685+
if (tokenCount + countFileToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
686+
tokenCount += countFileToken;
687+
neighborFileContext += `${commentPrefix}<filename>neighbor files:\n\n ${neighborFiles[0].file}\n\n`;
688+
neighborFileContext += `${neighborFiles[0].text}\n\n\n\n`;
689+
}
690+
}
691+
} catch (error) {
692+
logger.channel()?.error("Error creating neighbor file context:", error);
693+
}
653694
}
654-
}
655695

656-
let recentEditContext = "";
657-
if (tokenCount < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
658-
recentEditContext = await createRecentEditContext(recentEdits, filePath);
696+
logger.channel()?.debug("Complete token:", tokenCount);
659697

660-
const countRecentToken = countTokens(recentEditContext);
661-
if (tokenCount + countRecentToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
662-
tokenCount += countRecentToken;
698+
let prompt = "";
699+
let completeModel: string = DevChatConfig.getInstance().get("complete_model");
700+
if (!completeModel) {
701+
completeModel = "nvidia/starcoder2:15b";
702+
}
703+
if (completeModel.indexOf("deepseek") > -1) {
704+
prompt = "<｜fim▁begin｜>" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix + "<｜fim▁hole｜>" + suffix + "<｜fim▁end｜>";
705+
} else if (completeModel.indexOf("starcoder") > -1) {
706+
prompt = "<fim_prefix>" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix + "<fim_suffix>" + suffix + "<fim_middle>";
707+
} else if (completeModel.indexOf("codestral") > -1) {
708+
prompt = "<s>[SUFFIX]" + suffix + "[PREFIX]" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix;
663709
} else {
664-
recentEditContext = "";
710+
prompt = "<fim_prefix>" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix + "<fim_suffix>" + suffix + "<fim_middle>";
665711
}
666-
}
667712

668-
let neighborFileContext = "";
669-
if (tokenCount < 200) {
670-
const neighborFiles = await findNeighborFileContext(filePath, fileContent, line, column);
671-
if (neighborFiles.length > 0) {
672-
const countFileToken = countTokens(neighborFiles[0].text);
673-
if (tokenCount + countFileToken < DevChatConfig.getInstance().get("complete_context_limit", 3000)) {
674-
tokenCount += countFileToken;
675-
neighborFileContext += `${commentPrefix}<filename>neighbor files:\n\n ${neighborFiles[0].file}\n\n`;
676-
neighborFileContext += `${neighborFiles[0].text}\n\n\n\n`;
677-
}
678-
}
679-
}
680-
681-
logger.channel()?.debug("Complete token:", tokenCount);
682-
683-
let prompt = "";
684-
let completeModel: string = DevChatConfig.getInstance().get("complete_model");
685-
if (!completeModel) {
686-
completeModel = "nvidia/starcoder2:15b";
687-
}
688-
if (completeModel.indexOf("deepseek") > -1) {
689-
prompt = "<｜fim▁begin｜>" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix + "<｜fim▁hole｜>" + suffix + "<｜fim▁end｜>";
690-
} else if (completeModel.indexOf("starcoder") > -1) {
691-
prompt = "<fim_prefix>" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix + "<fim_suffix>" + suffix + "<fim_middle>";
692-
} else if (completeModel.indexOf("codestral") > -1) {
693-
prompt = "<s>[SUFFIX]" + suffix + "[PREFIX]" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix;
694-
} else {
695-
prompt = "<fim_prefix>" + taskDescriptionContextWithCommentPrefix + neighborFileContext + recentEditContext + symbolContext + callDefContext + similarBlockContext + gitDiffContext + `${commentPrefix}<filename>${filePath}\n\n` + prefix + "<fim_suffix>" + suffix + "<fim_middle>";
713+
return prompt;
714+
} catch (error) {
715+
logger.channel()?.error("Error in createPrompt:", error);
716+
return undefined;
696717
}
697-
698-
return prompt;
699718
}
700719

701720
function findImportTypeDefine(filePath: string, fileContent: string, node: Parser.SyntaxNode) {

0 commit comments

Comments
 (0)