feat: allow using pre-generated .rs files for protocol buffers. (#553)

When YARA-X is build, the .proto files that describe modules are automatically converted into .rs files. This relies on using the protobuf_codegen create or the protoc compiler. However, in certain cases is useful to have pre-generated .rs files that can be used during compilation, instead of having to convert the .proto files into .rs files all the time.

This PR introduces such feature. The default behavior is still generating the .rs files with every build, but now you can use pre-generated files in contained in the protos/generated folder, by disabling the generate-proto-code feature (which is enabled by default).

Author: plusvic

.github/workflows/release.yaml

@@ -230,10 +230,11 @@ jobs:
 
     - name: Publish crate
       env:
-        # Don't re-generate the files `modules.rs` and `add_modules.rs`, use
-        # the ones in the repository. These files can't be regenerated because
-        # `cargo publish` checks that the repository doesn't change during
-        # the build. See: https://github.com/rust-lang/cargo/pull/5584
+        # Don't re-generate `modules.rs`, `add_modules.rs` and the files in
+        # `protos/generated`. Use the ones in the repository. These files can't
+        # be regenerated because `cargo publish` checks that the repository
+        # doesn't change during the build.
+        # See: https://github.com/rust-lang/cargo/pull/5584
         YRX_REGENERATE_MODULES_RS: "no"
       run: |
         cargo login ${{ secrets.CRATES_IO_API_TOKEN }}

lib/Cargo.toml

@@ -49,18 +49,23 @@ exact-atoms = []
 # the fast regexp matching mechanism for testing purposes.
 fast-regexp = []
 
+# Enables the generation of Rust code from .proto files. This feature is enabled
+# by default. When disabled, the build process will use pre-generated Rust files
+# found in the source tree, instead of generating them at build time.
+generate-proto-code = ["dep:protobuf-codegen", "dep:protobuf-parse"]
+
 # Uses the `inventory` crate (https://github.com/dtolnay/inventory) instead
-# of `linkme`  (https://github.com/dtolnay/linkme) for tracking WASM exports.
+# of `linkme` (https://github.com/dtolnay/linkme) for tracking WASM exports.
 #
 # This feature is disabled by default.
 inventory = ["dep:inventory"]
 
 # Whether to use protoc for parsing and compiling .proto files. By default,
 # .proto files are parsed and compiled by the pure-Rust compiler implemented
-# by the `rust-protobuf` crate. With this feature you can change this behavior
-# and use protoc, the official Protocol Buffer compiler. You'll need to have
-# protoc installed in your system, together with the protoc-gen-rust plugin.
-# Follow the instructions in: https://lib.rs/crates/protobuf-codegen3
+# by the `protobuf_codegen` crate. With this feature you can change this
+# behavio and use `protoc`, the official Protocol Buffer compiler. You'll need
+# to have protoc installed in your system, together with the protoc-gen-rust
+# plugin. Follow the instructions in: https://lib.rs/crates/protobuf-codegen3
 protoc = []
 
 # Uses the `linkme` crate (https://github.com/dtolnay/linkme) instead
@@ -246,6 +251,7 @@ default = [
     "constant-folding",
     "default-modules",
     "fast-regexp",
+    "generate-proto-code",
     "linkme",
 ]
 
@@ -317,8 +323,8 @@ lingua = { version = "1.6.2", optional = true, default-features = false, feature
 anyhow = { workspace = true }
 globwalk = { workspace = true }
 protobuf = { workspace = true }
-protobuf-codegen = { workspace = true }
-protobuf-parse = { workspace = true }
+protobuf-codegen = { workspace = true, optional = true }
+protobuf-parse = { workspace = true, optional = true }
 yara-x-proto = { workspace = true }
 
 [dev-dependencies]

lib/build.rs

@@ -1,16 +1,16 @@
-use anyhow::Context;
+#[cfg(feature = "generate-proto-code")]
 use protobuf::descriptor::FileDescriptorProto;
-use std::fs::File;
-use std::io::Write;
-use std::path::PathBuf;
-use std::{env, fs};
 
-use protobuf_codegen::Codegen;
-use protobuf_parse::Parser;
+#[cfg(feature = "generate-proto-code")]
+fn generate_module_files(proto_files: Vec<FileDescriptorProto>) {
+    use std::fs::File;
+    use std::io::Write;
+    use std::path::PathBuf;
+    use yara_x_proto::exts::module_options as yara_module_options;
 
-use yara_x_proto::exts::module_options as yara_module_options;
+    println!("cargo:rerun-if-changed=src/modules/add_modules.rs");
+    println!("cargo:rerun-if-changed=src/modules/modules.rs");
 
-fn generate_module_files(proto_files: Vec<FileDescriptorProto>) {
     let mut modules = Vec::new();
     // Look for .proto files that describe a YARA module. A proto that
     // describes a YARA module has yara.module_options, like...
@@ -141,16 +141,14 @@ add_module!(modules, "{name}", {proto_mod}, "{root_message}", {rust_mod_name}, {
     write!(add_modules_rs, "\n}}").unwrap();
 }
 
-fn main() {
-    println!("cargo:rerun-if-changed=src/modules/protos");
-    println!("cargo:rerun-if-changed=src/modules/add_modules.rs");
-    println!("cargo:rerun-if-changed=src/modules/modules.rs");
+#[cfg(feature = "generate-proto-code")]
+fn generate_proto_code() {
+    use anyhow::Context;
+    use std::path::PathBuf;
+    use std::{env, fs};
 
-    if !cfg!(feature = "inventory") && !cfg!(feature = "linkme") {
-        panic!(
-            "either the `inventory` feature or the `linkme` feature must be enabled."
-        );
-    }
+    use protobuf_codegen::Codegen;
+    use protobuf_parse::Parser;
 
     let mut proto_compiler = Codegen::new();
     let mut proto_parser = Parser::new();
@@ -163,6 +161,8 @@ fn main() {
         proto_parser.pure();
     }
 
+    println!("cargo:rerun-if-changed=src/modules/protos");
+
     proto_compiler.cargo_out_dir("protos").include("./src/modules/protos");
     proto_parser.include("./src/modules/protos");
 
@@ -235,28 +235,53 @@ fn main() {
     // Generate .rs files for .proto files in src/modules/protos
     proto_compiler.run_from_script();
 
-    // Decide whether the `modules.rs` and `add_modules.rs` files should be
-    // re-generated. By default, they will be re-generated.
-    let mut regenerate_modules_rs = true;
+    // Decide whether `modules.rs`, `add_modules.rs` and the content of the
+    // `protos/generated` directory should be re-generated. By default, they
+    // will be re-generated.
+    let mut regenerate = true;
 
     // If the environment variable `YRX_REGENERATE_MODULES_RS` is present, the
     // files won't be re-generated if the value is "false", "no" or "0". Any
     // other value will re-generate the files.
     if let Ok(env_var) = env::var("YRX_REGENERATE_MODULES_RS") {
-        regenerate_modules_rs =
-            env_var != "false" && env_var != "no" && env_var != "0";
+        regenerate = env_var != "false" && env_var != "no" && env_var != "0";
     }
 
     // Also, don't re-generate the files if `DOCS_RS` is defined. This is
     // because doc.rs puts the source code in a read-only file system, and
     // we can't modify the files.
     if env::var("DOCS_RS").is_ok() {
-        regenerate_modules_rs = false;
+        regenerate = false;
     }
 
-    if regenerate_modules_rs {
+    if regenerate {
         generate_module_files(
             proto_parser.file_descriptor_set().unwrap().file,
         );
+
+        let out_dir = env::var("OUT_DIR").unwrap();
+        let src_dir = PathBuf::from("src/modules/protos/generated");
+        let _ = fs::create_dir_all(&src_dir);
+
+        for entry in globwalk::glob(format!("{}/protos/*.rs", out_dir))
+            .unwrap()
+            .flatten()
+        {
+            let path = entry.path();
+            let file_name = path.file_name().unwrap();
+            let dest = src_dir.join(file_name);
+            fs::copy(path, dest).unwrap();
+        }
+    }
+}
+
+fn main() {
+    if !cfg!(feature = "inventory") && !cfg!(feature = "linkme") {
+        panic!(
+            "either the `inventory` feature or the `linkme` feature must be enabled."
+        );
     }
+
+    #[cfg(feature = "generate-proto-code")]
+    generate_proto_code();
 }

lib/src/modules/mod.rs

@@ -7,7 +7,11 @@ use rustc_hash::FxHashMap;
 use thiserror::Error;
 
 pub mod protos {
+    #[cfg(feature = "generate-proto-code")]
     include!(concat!(env!("OUT_DIR"), "/protos/mod.rs"));
+
+    #[cfg(not(feature = "generate-proto-code"))]
+    include!("protos/generated/mod.rs");
 }
 
 #[cfg(test)]