Serialization support (#11)

* Serialization support

* Automated pre-commit update

* error handling

* Reformat, use an arc for buffers, add compression

* Use the same build id across threads

* Automated pre-commit update

* Add logging around serialization

* Make methods public

* Fix test

* make all tests pass

---------

Co-authored-by: eacodegen <[email protected]>

Author: bagel897

Cargo.lock

@@ -7,13 +7,15 @@ edition = "2021"
 clap = { version = "4.5.28", features = ["derive"] }
 codegen-sdk-analyzer = { path = "codegen-sdk-analyzer" }
 codegen-sdk-cst = { path = "codegen-sdk-cst" , features = ["typescript", "javascript", "tsx", "jsx"]}
-codegen-sdk-common = { path = "codegen-sdk-common" }
+codegen-sdk-common = { path = "codegen-sdk-common"}
 crossbeam = "0.8.4"
 glob = "0.3.2"
 env_logger = { workspace = true }
 log = { workspace = true }
 rayon = { workspace = true}
 sysinfo = "0.33.1"
+rkyv.workspace = true
+
 [workspace]
 members = [
   "codegen-sdk-analyzer",
@@ -38,3 +40,4 @@ convert_case = "0.7.1"
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.138"
 anyhow = { version = "1.0.95", features = ["backtrace"] }
+rkyv = { version = "0.8.10", features = ["bytes-1","pointer_width_64"] }

Cargo.toml

@@ -21,6 +21,12 @@ convert_case = { workspace = true }
 tree-sitter-query = {git = "https://github.com/tree-sitter-grammars/tree-sitter-query", optional = true}
 tree-sitter-language = "0.1.4"
 phf = { version = "0.11.3", features = ["macros"] }
+rkyv = { workspace = true }
+xdg = "2.5.2"
+base64 = "0.22.1"
+buildid = "1.0.3"
+sha2 = "0.10.8"
+zstd = { version = "0.13.2", features = ["zstdmt"] }
 [features]
 python = ["dep:tree-sitter-python"]
 json = ["dep:tree-sitter-json"]

codegen-sdk-ast/src/main.rs

@@ -1,4 +1,5 @@
 use std::backtrace::Backtrace;
+
 use thiserror::Error;
 #[derive(Debug, Error)]
 pub enum ParseError {
@@ -27,4 +28,6 @@ pub enum ParseError {
         node_type: String,
         backtrace: Backtrace,
     },
+    #[error("Failed to serialize: {0}")]
+    Serialize(#[from] rkyv::rancor::Error),
 }

codegen-sdk-common/Cargo.toml

@@ -1,9 +1,10 @@
+use convert_case::{Case, Casing};
+use tree_sitter::Parser;
+
 use crate::{
     errors::ParseError,
     parser::{Node, parse_node_types},
 };
-use convert_case::{Case, Casing};
-use tree_sitter::Parser;
 pub struct Language {
     pub name: &'static str,
     pub struct_name: &'static str,

codegen-sdk-common/src/errors.rs

@@ -13,3 +13,6 @@ pub mod parser;
 #[macro_use]
 extern crate lazy_static;
 pub mod naming;
+mod point;
+pub use point::Point;
+pub mod serialize;

codegen-sdk-common/src/language.rs

@@ -1,6 +1,7 @@
+use std::debug_assert;
+
 use convert_case::{Case, Casing};
 use phf::phf_map;
-use std::debug_assert;
 
 static MAPPINGS: phf::Map<char, &'static str> = phf_map! {
     '-' => "Minus",

codegen-sdk-common/src/lib.rs

@@ -0,0 +1,15 @@
+use rkyv::{Archive, Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Archive, Deserialize, Serialize)]
+pub struct Point {
+    pub row: usize,
+    pub column: usize,
+}
+impl From<tree_sitter::Point> for Point {
+    fn from(value: tree_sitter::Point) -> Self {
+        Point {
+            row: value.row,
+            column: value.column,
+        }
+    }
+}

codegen-sdk-common/src/naming.rs

@@ -0,0 +1,59 @@
+use std::{
+    fs::File,
+    io::{BufReader, BufWriter, Read},
+    path::PathBuf,
+};
+
+use base64::{Engine as _, engine::general_purpose::URL_SAFE};
+use bytes::Bytes;
+use rkyv::ser::writer::IoWriter;
+use sha2::{Digest, Sha256};
+use zstd::stream::AutoFinishEncoder;
+
+use crate::ParseError;
+pub struct Cache {
+    base_dir: PathBuf,
+    build_id: String,
+}
+impl Cache {
+    pub fn new() -> anyhow::Result<Self> {
+        let xdg_dirs = xdg::BaseDirectories::with_prefix("codegen")?;
+        let build_id = buildid::build_id().unwrap();
+        let encoded_build_id = URL_SAFE.encode(build_id);
+        xdg_dirs.create_cache_directory(&encoded_build_id)?;
+        Ok(Self {
+            base_dir: xdg_dirs.get_cache_home(),
+            build_id: encoded_build_id,
+        })
+    }
+    pub fn get_path(&self, path: &PathBuf) -> PathBuf {
+        let mut hasher = Sha256::new();
+        hasher.update(path.as_os_str().to_str().unwrap().as_bytes());
+        let path_hash = hasher.finalize();
+        self.base_dir
+            .join(format!("{}/{}", self.build_id, URL_SAFE.encode(path_hash)))
+    }
+    pub fn read_entry(&self, path: &PathBuf) -> Result<Bytes, ParseError> {
+        let file = File::open(path)?;
+        let mut buf = Vec::new();
+        let mut reader = zstd::Decoder::new(BufReader::new(file))?;
+        reader.read_to_end(&mut buf)?;
+        Ok(Bytes::from(buf))
+    }
+    pub fn get_writer(
+        &self,
+        path: &PathBuf,
+    ) -> Result<
+        IoWriter<
+            AutoFinishEncoder<
+                BufWriter<File>,
+                Box<dyn FnMut(Result<BufWriter<File>, std::io::Error>) + Send>,
+            >,
+        >,
+        ParseError,
+    > {
+        let file = File::create(path)?;
+        let writer = zstd::Encoder::new(BufWriter::new(file), 1)?.auto_finish();
+        Ok(IoWriter::new(writer))
+    }
+}