feat: support Substrait version 0.18 (#56)

BREAKING CHANGE:
 - buffer passed to substrait_validator::parse() must now be Clone, in
   order to be able to parse it as both substrait.Plan and
   substrait.PlanVersion (if the former fails)
 - substrait_validator::version() and substrait_validator::substrait_version()
   now return semver::Version objects rather than strings

Author: jvanstraten

Cargo.lock

@@ -27,7 +27,8 @@ older version. Refer to the table below for the version compatibility matrix.
 
 | Substrait...   | ... is supported by validator ...    |
 | -------------- | ------------------------------------ |
-| 0.9.x - 0.12.x | current version                      |
+| 0.18.x         | current version                      |
+| 0.9.x - 0.17.x | 0.0.8                                |
 | 0.7.x - 0.8.x  | 0.0.7                                |
 | 0.5.x - 0.6.x  | 0.0.6                                |
 | 0.3.x - 0.4.x  | 0.0.4 - 0.0.5                        |

README.md

@@ -833,7 +833,7 @@ pub extern "C" fn substrait_validator_export_proto(
 pub extern "C" fn substrait_validator_version() -> *const libc::c_char {
     static VERSION: once_cell::sync::OnceCell<std::ffi::CString> = once_cell::sync::OnceCell::new();
     VERSION
-        .get_or_init(|| std::ffi::CString::new(substrait_validator::version()).unwrap())
+        .get_or_init(|| std::ffi::CString::new(substrait_validator::version().to_string()).unwrap())
         .as_bytes_with_nul()
         .as_ptr() as *const libc::c_char
 }
@@ -843,7 +843,9 @@ pub extern "C" fn substrait_validator_version() -> *const libc::c_char {
 pub extern "C" fn substrait_validator_substrait_version() -> *const libc::c_char {
     static VERSION: once_cell::sync::OnceCell<std::ffi::CString> = once_cell::sync::OnceCell::new();
     VERSION
-        .get_or_init(|| std::ffi::CString::new(substrait_validator::substrait_version()).unwrap())
+        .get_or_init(|| {
+            std::ffi::CString::new(substrait_validator::substrait_version().to_string()).unwrap()
+        })
         .as_bytes_with_nul()
         .as_ptr() as *const libc::c_char
 }

c/src/lib.rs

@@ -31,9 +31,14 @@ TEST(BasicTest, BasicTest) {
   ASSERT_NE(data_ptr, nullptr);
   EXPECT_GT(data_size, 0);
   EXPECT_EQ(strlen(reinterpret_cast<const char *>(data_ptr)), data_size);
-  EXPECT_EQ(reinterpret_cast<const char *>(data_ptr),
-            std::string("Error at plan: failed to decode Protobuf message: "
-                        "invalid wire type value: 7 (code 1001)\n"));
+  EXPECT_EQ(
+      reinterpret_cast<const char *>(data_ptr),
+      std::string("Error at plan: failed to parse as substrait.Plan: "
+                  "failed to decode Protobuf message: "
+                  "invalid wire type value: 7 (code 1001) (code 1001)\n"
+                  "Error at plan: failed to parse as substrait.PlanVersion: "
+                  "failed to decode Protobuf message: "
+                  "invalid wire type value: 7 (code 1001) (code 1001)\n"));
 
   // Free the buffer.
   substrait_validator_free_exported(data_ptr);

c/tests/test.cc

@@ -263,14 +263,16 @@ fn substrait_validator(_py: Python, m: &PyModule) -> PyResult<()> {
     #[pyfn(m)]
     #[pyo3(name = "get_version")]
     fn get_version_py(py: Python) -> PyResult<PyObject> {
-        Ok(substrait_validator::version().to_object(py))
+        Ok(substrait_validator::version().to_string().to_object(py))
     }
 
     /// Returns the version of Substrait that the validator was built against.
     #[pyfn(m)]
     #[pyo3(name = "get_substrait_version")]
     fn get_substrait_version_py(py: Python) -> PyResult<PyObject> {
-        Ok(substrait_validator::substrait_version().to_object(py))
+        Ok(substrait_validator::substrait_version()
+            .to_string()
+            .to_object(py))
     }
 
     m.add_class::<Config>()?;

py/src/lib.rs

@@ -2,6 +2,10 @@
 
 BASIC_PLAN = """
 {
+ "version": {
+  "minor": 1,
+  "producer": "validator-test"
+ },
  "extensionUris": [],
  "extensions": [],
  "relations": [

py/tests/data.py

@@ -124,6 +124,9 @@ def resolver(s):
     # Disable "not yet implemented" warnings.
     config.override_diagnostic_level(1, "info", "info")
 
+    # Disable versioning warnings.
+    config.override_diagnostic_level(7, "info", "info")
+
     # Disable missing root relation error, so we don't have to supply one.
     config.override_diagnostic_level(5001, "info", "info")
 
@@ -136,12 +139,16 @@ def resolver(s):
 
     sv.check_plan_valid(
         {
+            "version": {
+                "minor": 1,
+                "producer": "validator-test",
+            },
             "extensionUris": [
                 {
                     "extension_uri_anchor": 1,
                     "uri": "test:hello",
                 }
-            ]
+            ],
         },
         config,
     )
@@ -152,12 +159,16 @@ def resolver(s):
     ):
         sv.check_plan_valid(
             {
+                "version": {
+                    "minor": 1,
+                    "producer": "validator-test",
+                },
                 "extensionUris": [
                     {
                         "extension_uri_anchor": 1,
                         "uri": "test:bye",
                     }
-                ]
+                ],
             },
             config,
         )

py/tests/test_api.py

@@ -85,6 +85,9 @@ float-pretty-print = "0.1"
 base64 = "0.13"
 percent-encoding = "2.1"
 
+# Used for representing and checking Substrait versions.
+semver = "1.0"
+
 [build-dependencies]
 
 # Used for generating Rust structs from the protobuf definitions.

rs/Cargo.toml

@@ -169,6 +169,8 @@ pub mod input;
 
 mod util;
 
+use std::str::FromStr;
+
 use strum::IntoEnumIterator;
 
 // Aliases for common types used on the crate interface.
@@ -182,7 +184,7 @@ pub use output::parse_result::ParseResult;
 pub use output::parse_result::Validity;
 
 /// Validates the given substrait.Plan message and returns the parse tree.
-pub fn parse<B: prost::bytes::Buf>(buffer: B, config: &Config) -> ParseResult {
+pub fn parse<B: prost::bytes::Buf + Clone>(buffer: B, config: &Config) -> ParseResult {
     parse::parse(buffer, config)
 }
 
@@ -192,12 +194,31 @@ pub fn iter_diagnostics() -> impl Iterator<Item = Classification> {
 }
 
 /// Returns the version of the validator.
-pub fn version() -> &'static str {
-    env!("CARGO_PKG_VERSION")
+pub fn version() -> semver::Version {
+    semver::Version::from_str(env!("CARGO_PKG_VERSION")).expect("invalid embedded crate version")
 }
 
 /// Returns the version of Substrait that this version of the validator was
 /// built against.
-pub fn substrait_version() -> &'static str {
-    include_str!("resources/substrait-version")
+pub fn substrait_version() -> semver::Version {
+    semver::Version::from_str(include_str!("resources/substrait-version"))
+        .expect("invalid embedded Substrait version")
+}
+
+/// Returns the Substrait version requirement for plans to be known to be
+/// supported.
+pub fn substrait_version_req() -> semver::VersionReq {
+    let version = substrait_version();
+    if version.major == 0 {
+        semver::VersionReq::parse(&format!("={}.{}", version.major, version.minor)).unwrap()
+    } else {
+        semver::VersionReq::parse(&format!("={}", version.major)).unwrap()
+    }
+}
+
+/// Returns the Substrait version requirement for plans to possibly be
+/// supported.
+pub fn substrait_version_req_loose() -> semver::VersionReq {
+    let version = substrait_version();
+    semver::VersionReq::parse(&format!("={}", version.major)).unwrap()
 }

rs/src/lib.rs

@@ -174,6 +174,9 @@ pub enum Classification {
     #[strum(props(Description = "deprecation"))]
     Deprecation = 6,
 
+    #[strum(props(HiddenDescription = "versioning"))]
+    Versioning = 7,
+
     #[strum(props(HiddenDescription = "experimental"))]
     Experimental = 999,