Add PHP Language Support (#51)

Author: WSHAPER

codeqai/app.py

@@ -8,14 +8,15 @@
 from rich.console import Console
 from rich.markdown import Markdown
 from rich.syntax import Syntax
+from pygments.lexers import PhpLexer
 from streamlit.web import cli as stcli
 from yaspin import yaspin
 
 from codeqai import codeparser, repo, utils
 from codeqai.bootstrap import bootstrap
 from codeqai.cache import create_cache_dir, get_cache_path, save_vector_cache
 from codeqai.config import create_config, get_config_path, load_config
-from codeqai.constants import DistillationMode, EmbeddingsModel, LlmHost
+from codeqai.constants import DistillationMode, EmbeddingsModel, Language, LlmHost
 from codeqai.dataset_extractor import DatasetExtractor
 from codeqai.embeddings import Embeddings
 from codeqai.vector_store import VectorStore
@@ -243,14 +244,26 @@ def run():
                         doc.metadata["filename"], doc.page_content
                     )
 
-                    syntax = Syntax(
-                        indentation + doc.page_content,
-                        language.value,
-                        theme="monokai",
-                        line_numbers=True,
-                        start_line=start_line,
-                        indent_guides=True,
-                    )
+                    # PHP needs startinline=True since code snippets don't have <?php tag
+                    if language == Language.PHP:
+                        lexer = PhpLexer(startinline=True)
+                        syntax = Syntax(
+                            indentation + doc.page_content,
+                            lexer=lexer,
+                            theme="monokai",
+                            line_numbers=True,
+                            start_line=start_line,
+                            indent_guides=True,
+                        )
+                    else:
+                        syntax = Syntax(
+                            indentation + doc.page_content,
+                            language.value,
+                            theme="monokai",
+                            line_numbers=True,
+                            start_line=start_line,
+                            indent_guides=True,
+                        )
                     print(
                         doc.metadata["filename"] + " -> " + doc.metadata["method_name"]
                     )

codeqai/codeparser.py

@@ -23,7 +23,7 @@ def parse_code_files_for_db(code_files: list[str]) -> list[Document]:
     documents = []
     code_splitter = None
     for code_file in code_files:
-        with open(code_file, "r", encoding="utf-8") as file:
+        with open(code_file, "r", encoding="utf-8", errors="replace") as file:
             file_bytes = file.read().encode()
             commit_hash = repo.get_commit_hash(code_file)
 
@@ -87,7 +87,7 @@ def parse_code_files_for_finetuning(
     output_tokens = 0
     documents = []
     for code_file in code_files:
-        with open(code_file, "r", encoding="utf-8") as file:
+        with open(code_file, "r", encoding="utf-8", errors="replace") as file:
             file_bytes = file.read().encode()
 
             file_extension = utils.get_file_extension(code_file)

codeqai/constants.py

@@ -17,6 +17,7 @@ class Language(Enum):
     LUA = "lua"
     HASKELL = "haskell"
     RUBY = "ruby"
+    PHP = "php"
     UNKNOWN = "unknown"
 
 

codeqai/repo.py

@@ -123,6 +123,7 @@ def get_commit_hash(file_path):
     "cdk.out",
     ".aws-sam",
     ".terraform",
+    "vendor",
 ]
 WHITELIST_FILES = [
     ".js",

codeqai/treesitter/__init__.py

@@ -12,3 +12,4 @@
 from codeqai.treesitter.treesitter_rs import TreesitterRust
 from codeqai.treesitter.treesitter_ts import TreesitterTypescript
 from codeqai.treesitter.treesitter_hs import TreesitterHaskell
+from codeqai.treesitter.treesitter_php import TreesitterPHP

codeqai/treesitter/treesitter_php.py

@@ -0,0 +1,13 @@
+from codeqai.constants import Language
+from codeqai.treesitter.treesitter import Treesitter
+from codeqai.treesitter.treesitter_registry import TreesitterRegistry
+
+
+class TreesitterPHP(Treesitter):
+    def __init__(self):
+        super().__init__(
+            Language.PHP, "method_declaration", "name", "comment"
+        )
+
+
+TreesitterRegistry.register_treesitter(Language.PHP, TreesitterPHP)

codeqai/utils.py

@@ -35,6 +35,7 @@ def get_programming_language(file_extension: str) -> Language:
         ".cs": Language.C_SHARP,
         ".hs": Language.HASKELL,
         ".rb": Language.RUBY,
+        ".php": Language.PHP,
     }
     return language_mapping.get(file_extension, Language.UNKNOWN)
 
@@ -76,6 +77,8 @@ def get_langchain_language(language: Language):
         return text_splitter.Language.HASKELL
     elif language == Language.RUBY:
         return text_splitter.Language.RUBY
+    elif language == Language.PHP:
+        return text_splitter.Language.PHP
     else:
         return None