Get rid of all transport types and settle on Protobuf (#25)

* Get rid of all transport types and settle on Protobuf

hope i don't regret this

* Update Cargo.toml

* Update agent.py

Author: joshuadavidthomas

.editorconfig

@@ -9,7 +9,7 @@ trim_trailing_whitespace = true
 [{,.}{j,J}ustfile]
 indent_size = 4
 
-[*.{py,rst,ini,md}]
+[*.{just,proto,py,rst,ini,md}]
 indent_size = 4
 
 [*.py]

.just/proto.just

@@ -22,16 +22,84 @@ fmt:
 # Generate protobuf code for both Rust and Python
 [no-cd]
 gen:
-  @just proto rust
-  @just proto py
+    @just proto rust
+    @just proto py
 
 # Generate protobuf code for Rust
 [no-cd]
 rust: check
-    cargo build -p djls-types
+    @just proto clean-rust
+    cargo build -p djls-ipc
+
+[private]
+clean-rust:
+    #!/usr/bin/env python3
+    from pathlib import Path
+    import shutil
+
+    target_dir = Path("{{ justfile_directory() }}/target")
+
+    if target_dir.exists():
+        for item in target_dir.rglob('*djls[_-]ipc*'):
+            if item.is_file():
+                item.unlink()
+            elif item.is_dir():
+                shutil.rmtree(item)
 
 # Generate protobuf code for Python
 [no-cd]
 py: check
-    protoc -I=proto --python_out=python/djls proto/*.proto
+    @just proto clean-py
+    protoc -I=proto \
+      --python_out=python/djls/proto \
+      --pyi_out=python/djls/proto \
+      proto/v1/*.proto
+    fd -t f "(_pb2\.py|_pb2\.pyi)$" python/djls/proto -x sed -i 's/from v1 import/from . import/g' {}
+    fd -t d "." python/djls/proto -x touch {}//__init__.py
+    @just proto py-add-warnings
+
+[private]
+clean-py:
+    #!/usr/bin/env python3
+    from pathlib import Path
+    import shutil
+
+    proto_dir = Path("{{ justfile_directory() }}/python/djls/proto")
+
+    for item in proto_dir.iterdir():
+        if item.is_file():
+            item.unlink()
+        elif item.is_dir():
+            shutil.rmtree(item)
+
+[private]
+py-add-warnings:
+    #!/usr/bin/env python3
+    from pathlib import Path
+
+    def create_warning(proto_file: str) -> str:
+        return f'''# WARNING: This file is generated by protobuf. DO NOT EDIT!
+    # Any changes made to this file will be overwritten when the protobuf files are regenerated.
+    # Source: {proto_file}
+
+    '''
+
+    proto_dir = Path("{{ justfile_directory() }}/python/djls/proto")
+    proto_source_dir = Path("{{ justfile_directory() }}/proto")
+
+    proto_sources = {
+        path.stem: path.relative_to(proto_source_dir)
+        for path in proto_source_dir.glob('**/*.proto')
+    }
+
+    for file_path in proto_dir.glob("**/*.py*"):  # Catches both .py and .pyi in all subdirs
+        proto_name = file_path.stem.removesuffix('_pb2')
+        source_proto = proto_sources.get(proto_name)
 
+        content = file_path.read_text()
+        if not content.startswith('# WARNING'):
+            warning = create_warning(
+                str(source_proto) if source_proto
+                else "generated by py-init"
+            )
+            file_path.write_text(warning + content)

Cargo.toml

@@ -9,7 +9,6 @@ djls-django = { path = "crates/djls-django" }
 djls-ipc = { path = "crates/djls-ipc" }
 djls-python = { path = "crates/djls-python" }
 djls-server = { path = "crates/djls-server" }
-djls-types = { path = "crates/djls-types" }
 djls-worker = { path = "crates/djls-worker" }
 
 anyhow = "1.0.94"

Justfile

@@ -8,6 +8,9 @@ mod proto ".just/proto.just"
 default:
     @just --list
 
+clean:
+    rm -rf target/
+
 # run pre-commit on all files
 lint:
     @just --fmt

crates/djlc-cli/src/main.rs

@@ -1,5 +1,7 @@
 use clap::{Args, Parser, Subcommand};
-use djls_ipc::{PythonProcess, Transport};
+use djls_ipc::v1::*;
+use djls_ipc::{ProcessError, PythonProcess, TransportError};
+use std::ffi::OsStr;
 use std::time::Duration;
 
 #[derive(Debug, Parser)]
@@ -41,8 +43,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     match cli.command {
         Commands::Serve(opts) => {
-            let python =
-                PythonProcess::new("djls.lsp", Transport::Json, opts.health_check_interval())?;
+            println!("Starting LSP server...");
+            let python = PythonProcess::new::<Vec<&OsStr>, &OsStr>(
+                "djls.agent",
+                None,
+                opts.health_check_interval(),
+            )?;
+            println!("LSP server started, beginning to serve...");
             djls_server::serve(python).await?
         }
     }

crates/djls-django/src/apps.rs

@@ -1,5 +1,5 @@
-use djls_ipc::{JsonResponse, PythonProcess, TransportError, TransportMessage, TransportResponse};
-use serde::Deserialize;
+use djls_ipc::v1::*;
+use djls_ipc::{ProcessError, PythonProcess};
 use std::fmt;
 
 #[derive(Debug)]
@@ -20,23 +20,6 @@ impl fmt::Display for App {
 #[derive(Debug, Default)]
 pub struct Apps(Vec<App>);
 
-#[derive(Debug, Deserialize)]
-struct InstalledAppsCheck {
-    has_app: bool,
-}
-
-impl TryFrom<JsonResponse> for InstalledAppsCheck {
-    type Error = TransportError;
-
-    fn try_from(response: JsonResponse) -> Result<Self, Self::Error> {
-        response
-            .data()
-            .clone()
-            .ok_or_else(|| TransportError::Process("No data in response".to_string()))
-            .and_then(|data| serde_json::from_value(data).map_err(TransportError::Json))
-    }
-}
-
 impl Apps {
     pub fn from_strings(apps: Vec<String>) -> Self {
         Self(apps.into_iter().map(App).collect())
@@ -54,18 +37,21 @@ impl Apps {
         self.apps().iter()
     }
 
-    pub fn check_installed(python: &mut PythonProcess, app: &str) -> Result<bool, TransportError> {
-        let message = TransportMessage::Json("installed_apps_check".to_string());
-        let response = python.send(message, Some(vec![app.to_string()]))?;
-        match response {
-            TransportResponse::Json(json_str) => {
-                let json_response: JsonResponse = serde_json::from_str(&json_str)?;
-                let result = InstalledAppsCheck::try_from(json_response)?;
-                Ok(result.has_app)
-            }
-            _ => Err(TransportError::Process(
-                "Unexpected response type".to_string(),
+    pub fn check_installed(python: &mut PythonProcess, app: &str) -> Result<bool, ProcessError> {
+        let request = messages::Request {
+            command: Some(messages::request::Command::CheckAppInstalled(
+                check::AppInstalledRequest {
+                    app_name: app.to_string(),
+                },
             )),
+        };
+
+        let response = python.send(request).map_err(ProcessError::Transport)?;
+
+        match response.result {
+            Some(messages::response::Result::CheckAppInstalled(response)) => Ok(response.passed),
+            Some(messages::response::Result::Error(e)) => Err(ProcessError::Health(e.message)),
+            _ => Err(ProcessError::Response),
         }
     }
 }

crates/djls-django/src/django.rs

@@ -1,63 +1,26 @@
-use crate::apps::Apps;
 use crate::gis::{check_gis_setup, GISError};
-use crate::templates::TemplateTags;
-use djls_ipc::{JsonResponse, PythonProcess, TransportError, TransportMessage, TransportResponse};
+use djls_ipc::v1::*;
+use djls_ipc::{ProcessError, PythonProcess, TransportError};
 use djls_python::{ImportCheck, Python};
-use serde::Deserialize;
 use std::fmt;
 
 #[derive(Debug)]
 pub struct DjangoProject {
     py: Python,
     python: PythonProcess,
-    settings_module: String,
-    installed_apps: Apps,
-    templatetags: TemplateTags,
-}
-
-#[derive(Debug, Deserialize)]
-struct DjangoSetup {
-    installed_apps: Vec<String>,
-    templatetags: TemplateTags,
-}
-
-impl DjangoSetup {
-    pub fn setup(python: &mut PythonProcess) -> Result<JsonResponse, ProjectError> {
-        let message = TransportMessage::Json("django_setup".to_string());
-        let response = python.send(message, None)?;
-        match response {
-            TransportResponse::Json(json_str) => {
-                let json_response: JsonResponse = serde_json::from_str(&json_str)?;
-                Ok(json_response)
-            }
-            _ => Err(ProjectError::Transport(TransportError::Process(
-                "Unexpected response type".to_string(),
-            ))),
-        }
-    }
+    version: String,
 }
 
 impl DjangoProject {
-    fn new(
-        py: Python,
-        python: PythonProcess,
-        settings_module: String,
-        installed_apps: Apps,
-        templatetags: TemplateTags,
-    ) -> Self {
+    fn new(py: Python, python: PythonProcess, version: String) -> Self {
         Self {
             py,
             python,
-            settings_module,
-            installed_apps,
-            templatetags,
+            version,
         }
     }
 
     pub fn setup(mut python: PythonProcess) -> Result<Self, ProjectError> {
-        let settings_module =
-            std::env::var("DJANGO_SETTINGS_MODULE").expect("DJANGO_SETTINGS_MODULE must be set");
-
         let py = Python::setup(&mut python)?;
 
         let has_django = ImportCheck::check(&mut python, Some(vec!["django".to_string()]))?;
@@ -74,45 +37,52 @@ impl DjangoProject {
             return Ok(Self {
                 py,
                 python,
-                settings_module,
-                installed_apps: Apps::default(),
-                templatetags: TemplateTags::default(),
+                version: String::new(),
             });
         }
 
-        let response = DjangoSetup::setup(&mut python)?;
-        let setup: DjangoSetup = response
-            .data()
-            .clone()
-            .ok_or_else(|| TransportError::Process("No data in response".to_string()))
-            .and_then(|data| serde_json::from_value(data).map_err(TransportError::Json))?;
+        let request = messages::Request {
+            command: Some(messages::request::Command::DjangoGetProjectInfo(
+                django::GetProjectInfoRequest {},
+            )),
+        };
 
-        Ok(Self::new(
+        let response = python
+            .send(request)
+            .map_err(|e| ProjectError::Transport(e))?;
+
+        let version = match response.result {
+            Some(messages::response::Result::DjangoGetProjectInfo(response)) => {
+                response.project.unwrap().version
+            }
+            Some(messages::response::Result::Error(e)) => {
+                return Err(ProjectError::Process(ProcessError::Health(e.message)));
+            }
+            _ => {
+                return Err(ProjectError::Process(ProcessError::Response));
+            }
+        };
+
+        Ok(Self {
             py,
             python,
-            settings_module,
-            Apps::from_strings(setup.installed_apps.to_vec()),
-            setup.templatetags,
-        ))
+            version,
+        })
     }
 
     pub fn py(&self) -> &Python {
         &self.py
     }
 
-    fn settings_module(&self) -> &String {
-        &self.settings_module
+    fn version(&self) -> &String {
+        &self.version
     }
 }
 
 impl fmt::Display for DjangoProject {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "Django Project")?;
-        writeln!(f, "Settings Module: {}", self.settings_module)?;
-        writeln!(f, "Installed Apps:")?;
-        write!(f, "{}", self.installed_apps)?;
-        writeln!(f, "Template Tags:")?;
-        write!(f, "{}", self.templatetags)?;
+        writeln!(f, "Version: {}", self.version)?;
         Ok(())
     }
 }
@@ -121,22 +91,18 @@ impl fmt::Display for DjangoProject {
 pub enum ProjectError {
     #[error("Django is not installed or cannot be imported")]
     DjangoNotFound,
-
     #[error("IO error: {0}")]
     Io(#[from] std::io::Error),
-
     #[error("GIS error: {0}")]
     Gis(#[from] GISError),
-
     #[error("JSON parsing error: {0}")]
     Json(#[from] serde_json::Error),
-
     #[error(transparent)]
     Packaging(#[from] djls_python::PackagingError),
-
+    #[error("Process error: {0}")]
+    Process(#[from] ProcessError),
     #[error(transparent)]
     Python(#[from] djls_python::PythonError),
-
     #[error("Transport error: {0}")]
     Transport(#[from] TransportError),
 }

crates/djls-django/src/gis.rs

@@ -1,5 +1,5 @@
 use crate::apps::Apps;
-use djls_ipc::{PythonProcess, TransportError};
+use djls_ipc::{ProcessError, PythonProcess, TransportError};
 use std::process::Command;
 
 pub fn check_gis_setup(python: &mut PythonProcess) -> Result<bool, GISError> {
@@ -17,10 +17,10 @@ pub fn check_gis_setup(python: &mut PythonProcess) -> Result<bool, GISError> {
 pub enum GISError {
     #[error("IO error: {0}")]
     Io(#[from] std::io::Error),
-
     #[error("JSON parsing error: {0}")]
     Json(#[from] serde_json::Error),
-
+    #[error("Process error: {0}")]
+    Process(#[from] ProcessError),
     #[error("Transport error: {0}")]
     Transport(#[from] TransportError),
 }