refactor: unify all parser into lang

Author: AsterDY

script/make_parser.sh

@@ -15,9 +15,9 @@
 
 root=$(dirname $(realpath $(dirname $0)))
 
-# make go_ast
-cd $root/src/compress/golang/plugin
-go build -o ../../../../tools/parser/go_ast .
+# # make go_ast
+# cd $root/src/compress/golang/plugin
+# go build -o ../../../../tools/parser/go_ast .
 
 
 # make lang

src/bin/cmd.rs

@@ -15,9 +15,8 @@
 use std::{env, panic, process, thread, time::Duration};
 
 use ABCoder::{
-    compress::{compress::compress_all, types::types::CodeCache},
-    config::CONFIG,
-    repo,
+    compress::compress::compress_all,
+    repo::{self, CompressOptions},
 };
 
 #[derive(Clone, Debug)]
@@ -36,6 +35,17 @@ struct CompressAction {
     export_compress: bool,
     force_update_ast: bool,
     not_load_external_symbol: bool,
+    no_need_comment: bool,
+}
+
+impl CompressAction {
+    fn to_compress_options(&self) -> CompressOptions {
+        CompressOptions {
+            export_compress: self.export_compress,
+            not_load_external_symbol: self.not_load_external_symbol,
+            no_need_comment: self.no_need_comment,
+        }
+    }
 }
 
 fn main() {
@@ -61,7 +71,9 @@ Action: compress
     compress: compress the repo. Including flags:
       --export-compress: export the compress result
       --force-update-ast: force parsing repo and merge the previous result 
-      --not-load-external-symbol: not load external external symbols to speed up parsing";
+      --not-load-external-symbol: not load external external symbols to speed up parsing
+      --no-need-comment: not need comment in symbol content (only works for Go now)
+";
 
 fn parse_options() -> Options {
     let args: Vec<String> = env::args().collect();
@@ -85,6 +97,9 @@ fn parse_options() -> Options {
                         "--not-load-external-symbol" => {
                             compress_action.not_load_external_symbol = true;
                         }
+                        "--no-need-comment" => {
+                            compress_action.no_need_comment = true;
+                        }
                         _ => {}
                     }
                 }
@@ -109,7 +124,7 @@ fn compress(repo_path: &String, cmp: &CompressAction) {
     // recoverable logic
     let run = || {
         // get the repo
-        let repo = repo::get_repo(repo_path, !cmp.not_load_external_symbol);
+        let repo = repo::get_repo(repo_path, &cmp.to_compress_options());
         if let Err(err) = repo {
             println!("get repo error: {:?}", err);
             process::exit(1);
@@ -144,7 +159,7 @@ fn compress(repo_path: &String, cmp: &CompressAction) {
 
 fn export_compress(repo_path: &String, cmp: &CompressAction) {
     // get the repo
-    let repo = repo::get_repo(repo_path, !cmp.not_load_external_symbol);
+    let repo = repo::get_repo(repo_path, &cmp.to_compress_options());
     if let Err(err) = repo {
         println!("get repo error: {:?}", err);
 
@@ -162,17 +177,17 @@ fn export_compress(repo_path: &String, cmp: &CompressAction) {
     println!("successfully compressed repo: {}", repo.id);
 }
 
-fn merge_compress(repo_path: &String, opts: &CompressAction) {
+fn merge_compress(repo_path: &String, cmp: &CompressAction) {
     // get old repo
-    let repo = repo::get_repo(repo_path, !opts.not_load_external_symbol);
+    let repo = repo::get_repo(repo_path, &cmp.to_compress_options());
     if let Err(err) = repo {
         println!("get repo error: {:?}", err);
         process::exit(1);
     }
     let mut repo = repo.unwrap();
 
     // parse new repo
-    let nrepo = repo::force_parse_repo(repo_path, !opts.not_load_external_symbol);
+    let nrepo = repo::force_parse_repo(repo_path, &cmp.to_compress_options());
     if let Err(err) = nrepo {
         println!("parse repo error: {:?}", err);
         process::exit(1);

src/compress/golang/mod.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 pub mod compress;
-pub mod golang;
 mod llm;
 pub mod parser;
 pub mod rust;

src/compress/golang/parser.rs

@@ -17,7 +17,7 @@ use std::path::{Path, PathBuf};
 use lazy_static::lazy_static;
 use serde::Deserialize;
 
-use crate::repo::{self};
+use crate::repo::{self, CompressOptions};
 
 #[derive(Debug)]
 pub enum Language {
@@ -185,6 +185,16 @@ pub enum ProgramLanguage {
     Unknown(String),
 }
 
+impl ProgramLanguage {
+    pub fn to_string(&self) -> String {
+        match self {
+            ProgramLanguage::Rust => "rust".to_string(),
+            ProgramLanguage::Go => "go".to_string(),
+            ProgramLanguage::Unknown(s) => s.to_string(),
+        }
+    }
+}
+
 fn decide_language(path: &str) -> ProgramLanguage {
     // scan root directory
     walkdir::WalkDir::new(path)
@@ -208,40 +218,22 @@ fn decide_language(path: &str) -> ProgramLanguage {
         .unwrap_or(ProgramLanguage::Unknown(path.to_string()))
 }
 
-pub fn parser_and_args<'a>(repo_path: &'a str, load_extern: bool) -> (String, Vec<String>) {
+pub fn parser_and_args<'a>(repo_path: &'a str, opts: &CompressOptions) -> (String, Vec<String>) {
     let lang = decide_language(repo_path);
-    let path = match lang {
-        ProgramLanguage::Go => go_ast_path(),
-        ProgramLanguage::Rust => rust_ast_path(),
-        _ => panic!("unsupported language"),
-    };
-    let args = match lang {
-        ProgramLanguage::Go => {
-            let mut args = vec!["--collect_comment".to_string()];
-            if load_extern {
-                args.push("--refer_code_depth=1".to_string());
-            }
-            if CONFIG.exclude_dirs.len() > 0 {
-                args.push(format!("--excludes={}", &CONFIG.exclude_dirs.join(",")));
-            }
-            args.push(repo_path.to_string());
-            args
-        }
-        ProgramLanguage::Rust => {
-            let mut args = vec![
-                "collect".to_string(),
-                "rust".to_string(),
-                repo_path.to_string(),
-            ];
-            for exclude in &CONFIG.exclude_dirs {
-                args.push(format!("--exclude={exclude}"));
-            }
-            if load_extern {
-                args.push("--load-external-symbol".to_string());
-            }
-            args
-        }
-        _ => panic!("unsupported language"),
-    };
+    let path = rust_ast_path();
+    let mut args = vec![
+        "collect".to_string(),
+        lang.to_string(),
+        repo_path.to_string(),
+    ];
+    for exclude in &CONFIG.exclude_dirs {
+        args.push(format!("--exclude={exclude}"));
+    }
+    if !opts.not_load_external_symbol {
+        args.push("--load-external-symbol".to_string());
+    }
+    if opts.no_need_comment {
+        args.push("--no-need-comment".to_string());
+    }
     (path, args)
 }

src/compress/golang/plugin/go.mod

@@ -31,6 +31,8 @@ import (
 type CollectOption struct {
 	LoadExternalSymbol bool
 	NeedStdSymbol      bool
+	NoNeedComment      bool
+	Language           lsp.Language
 	Excludes           []string
 }