allow attaching arbirtary data to decoder and pass it to the decoding function.

this way decoder can be stateful

Author: iliabylich

api.cpp

@@ -33,7 +33,7 @@ namespace lib_ruby_parser
 
         ParserOptions options(
             String::Copied("(eval)"),
-            MaybeDecoder(Decoder(nullptr)),
+            MaybeDecoder(Decoder(nullptr, nullptr)),
             MaybeTokenRewriter(TokenRewriter(nullptr)),
             true);
 

decoder.cpp

@@ -89,7 +89,7 @@ namespace lib_ruby_parser
     }
 
     // decoder
-    Decoder::Decoder(DecoderFunction f_) : f(f_) {}
+    Decoder::Decoder(DecoderFunction f_, void *state_) : f(f_), state(state_) {}
 
     // maybe decoder
     MaybeDecoder::MaybeDecoder(Decoder decoder_) : decoder(decoder_) {}
@@ -99,7 +99,7 @@ namespace lib_ruby_parser
     }
     MaybeDecoder MaybeDecoder::None()
     {
-        return MaybeDecoder(Decoder(nullptr));
+        return MaybeDecoder(Decoder(nullptr, nullptr));
     }
 
     bool MaybeDecoder::is_some() const
@@ -183,60 +183,57 @@ namespace lib_ruby_parser
 
     extern "C"
     {
-        DecoderBlob lib_ruby_parser__test__always_ok_decoder(void);
-        DecoderBlob lib_ruby_parser__test__always_err_decoder(void);
+        DecoderBlob lib_ruby_parser__test__always_ok_decoder(const char *output);
+        DecoderBlob lib_ruby_parser__test__always_err_decoder(const char *output);
     }
-    static DecoderResult call_decoder(Decoder decoder, String encoding, ByteList input)
+    static DecoderResult call_decoder(Decoder decoder, String encoding, ByteList input, void *state)
     {
         StringBlob encoding_blob = into_blob<String, StringBlob>(std::move(encoding));
         ByteListBlob input_blob = into_blob<ByteList, ByteListBlob>(std::move(input));
         // cast fn from
         // (String, ByteList) -> DecoderResult
         // to
         // (StringBlob, ByteListBlob) -> DecoderResultBlob
-        typedef DecoderResultBlob (*DecoderFunctionBlob)(StringBlob, ByteListBlob);
+        typedef DecoderResultBlob (*DecoderFunctionBlob)(StringBlob, ByteListBlob, void *);
         DecoderFunctionBlob fn_blob =
             reinterpret_cast<DecoderFunctionBlob>(
                 reinterpret_cast<void *>(
                     decoder.f));
-        DecoderResultBlob result_blob = fn_blob(encoding_blob, input_blob);
+        DecoderResultBlob result_blob = fn_blob(encoding_blob, input_blob, state);
         return from_blob<DecoderResultBlob, DecoderResult>(result_blob);
     }
 
     static void test_decoder_fields(void)
     {
         annotate_test;
 
-        const char *ALWAYS_OK = "always_ok";
-        const char *ALWAYS_ERR = "always_err";
-
         Decoder ok_decoder = from_blob<DecoderBlob, Decoder>(
-            lib_ruby_parser__test__always_ok_decoder());
-        DecoderResult ok_decoder_result = call_decoder(ok_decoder, String::Copied("utf-8"), ByteList::Copied("2 + 2", 5));
+            lib_ruby_parser__test__always_ok_decoder("anything"));
+        DecoderResult ok_decoder_result = call_decoder(ok_decoder, String::Copied("utf-8"), ByteList::Copied("2 + 2", 5), const_cast<char *>("foo"));
         assert_eq(ok_decoder_result.tag, DecoderResult::Tag::OK);
-        assert_byte_list(ok_decoder_result.as.ok, ALWAYS_OK);
+        assert_byte_list(ok_decoder_result.as.ok, "foo");
 
         Decoder err_decoder = from_blob<DecoderBlob, Decoder>(
-            lib_ruby_parser__test__always_err_decoder());
-        DecoderResult err_decoder_result = call_decoder(err_decoder, String::Copied("utf-8"), ByteList::Copied("2 + 2", 5));
+            lib_ruby_parser__test__always_err_decoder("anything"));
+        DecoderResult err_decoder_result = call_decoder(err_decoder, String::Copied("utf-8"), ByteList::Copied("2 + 2", 5), const_cast<char *>("bar"));
         assert_eq(err_decoder_result.as.err.tag, InputError::Tag::DECODING_ERROR);
-        assert_string_eq(err_decoder_result.as.err.as.unsupported_encoding, ALWAYS_ERR);
+        assert_string_eq(err_decoder_result.as.err.as.unsupported_encoding, "bar");
     }
 
     extern "C"
     {
-        MaybeDecoderBlob lib_ruby_parser__test__some_always_ok_decoder(void);
+        MaybeDecoderBlob lib_ruby_parser__test__some_always_ok_decoder(const char *output);
         MaybeDecoderBlob lib_ruby_parser__test__none_decoder(void);
     }
     static void test_maybe_decoder_fields(void)
     {
         annotate_test;
 
         Decoder always_ok_decoder = from_blob<DecoderBlob, Decoder>(
-            lib_ruby_parser__test__always_ok_decoder());
+            lib_ruby_parser__test__always_ok_decoder("foo"));
 
         MaybeDecoder some_decoder = from_blob<MaybeDecoderBlob, MaybeDecoder>(
-            lib_ruby_parser__test__some_always_ok_decoder());
+            lib_ruby_parser__test__some_always_ok_decoder("foo"));
         assert(some_decoder.is_some());
 
         assert_eq(some_decoder.decoder.f, always_ok_decoder.f);

decoder.hpp

@@ -71,13 +71,14 @@ namespace lib_ruby_parser
         static DecoderResult Err(InputError err);
     };
 
-    typedef DecoderResult (*DecoderFunction)(String, ByteList);
+    typedef DecoderResult (*DecoderFunction)(String, ByteList, void *);
     class Decoder
     {
     public:
         DecoderFunction f;
+        void *state;
 
-        explicit Decoder(DecoderFunction f);
+        explicit Decoder(DecoderFunction f, void *state);
     };
 
     class MaybeDecoder

ruby-parser-cpp/src/decoder.rs

@@ -52,53 +52,70 @@ pub extern "C" fn LIB_RUBY_PARSER_drop_decoder_result(decoder_result: *mut Decod
 #[repr(C)]
 #[derive(Debug)]
 pub struct Decoder {
-    pub f: extern "C" fn(encoding: StringBlob, input: ByteListBlob) -> DecoderResultBlob,
+    pub f: extern "C" fn(
+        encoding: StringBlob,
+        input: ByteListBlob,
+        state: *const std::ffi::c_void,
+    ) -> DecoderResultBlob,
+    pub state: *const std::ffi::c_void,
 }
 
 #[cfg(feature = "tests")]
 #[no_mangle]
-pub extern "C" fn lib_ruby_parser__test__always_ok_decoder() -> Decoder {
+pub extern "C" fn lib_ruby_parser__test__always_ok_decoder(output: *const u8) -> Decoder {
     #[no_mangle]
     pub extern "C" fn lib_ruby_parser__test__always_ok_decoder_fn(
         encoding: StringBlob,
         input: ByteListBlob,
+        state: *const std::ffi::c_void,
     ) -> DecoderResultBlob {
         // do cleanup
         drop(String::from(encoding));
         drop(Vec::<u8>::from(input));
-        // and return constant output
-        DecoderResultBlob::from(DecoderResult::Ok("always_ok".as_bytes().to_vec()))
+        // and return given output
+        let output = unsafe { std::ffi::CStr::from_ptr(state as *const i8) }
+            .to_str()
+            .unwrap();
+        DecoderResultBlob::from(DecoderResult::Ok(output.as_bytes().to_vec()))
     }
     Decoder {
         f: lib_ruby_parser__test__always_ok_decoder_fn,
+        state: output as *const std::ffi::c_void,
     }
 }
 
 #[cfg(feature = "tests")]
 #[no_mangle]
-pub extern "C" fn lib_ruby_parser__test__always_err_decoder() -> Decoder {
+pub extern "C" fn lib_ruby_parser__test__always_err_decoder(output: *const u8) -> Decoder {
     #[no_mangle]
     pub extern "C" fn lib_ruby_parser__test__always_err_decoder_fn(
         encoding: StringBlob,
         input: ByteListBlob,
+        state: *const std::ffi::c_void,
     ) -> DecoderResultBlob {
         // do cleanup
         drop(String::from(encoding));
         drop(Vec::<u8>::from(input));
-        // and return constant output
-        DecoderResultBlob::from(DecoderResult::Err(InputError::DecodingError(String::from(
-            "always_err",
-        ))))
+        // and return given output
+        let output = unsafe { std::ffi::CStr::from_ptr(state as *const i8) }
+            .to_str()
+            .unwrap();
+        DecoderResultBlob::from(DecoderResult::Err(InputError::DecodingError(
+            output.to_string(),
+        )))
     }
     Decoder {
         f: lib_ruby_parser__test__always_err_decoder_fn,
+        state: output as *const std::ffi::c_void,
     }
 }
 
 #[cfg(feature = "tests")]
 #[no_mangle]
-pub extern "C" fn lib_ruby_parser__test__some_always_ok_decoder() -> MaybeDecoderBlob {
-    MaybeDecoderBlob::from(Some(lib_ruby_parser__test__always_ok_decoder()))
+pub extern "C" fn lib_ruby_parser__test__some_always_ok_decoder(
+    output: *mut u8,
+) -> MaybeDecoderBlob {
+    MaybeDecoderBlob::from(Some(lib_ruby_parser__test__always_ok_decoder(output)))
 }
 
 #[cfg(feature = "tests")]

ruby-parser-cpp/src/lib.rs

@@ -6,7 +6,7 @@ macro_rules! blob_type {
         #[repr(C)]
         #[derive(Debug)]
         pub struct $c_type {
-            content: [u8; std::mem::size_of::<$rust_type>()],
+            pub(crate) content: [u8; std::mem::size_of::<$rust_type>()],
         }
 
         impl From<$rust_type> for $c_type {

ruby-parser-cpp/src/parser_options.rs

@@ -21,8 +21,9 @@ impl From<CParserOptions> for lib_ruby_parser::ParserOptions {
         } = options;
 
         let decoder = decoder.map(|decoder| {
+            let Decoder { f, state } = decoder;
             lib_ruby_parser::source::Decoder::new(Box::new(move |encoding, input| {
-                (decoder.f)(encoding.into(), input.into()).into()
+                f(encoding.into(), input.into(), state).into()
             }))
         });