Expect oplog data to be a JSON string

Author: simolus3

crates/core/src/json_writer.rs

@@ -21,7 +21,6 @@ mod error;
 mod ext;
 mod fix_data;
 mod json_merge;
-mod json_writer;
 mod kv;
 mod macros;
 mod migrations;

crates/core/src/lib.rs

@@ -1,15 +1,7 @@
-use core::assert_matches::debug_assert_matches;
-
 use alloc::borrow::Cow;
-use alloc::string::{String, ToString};
 use alloc::vec::Vec;
-use serde::{
-    de::{DeserializeSeed, Error, Visitor},
-    Deserialize,
-};
-use serde_json::value::RawValue;
+use serde::Deserialize;
 
-use crate::json_writer::JsonWriter;
 use crate::util::{deserialize_optional_string_to_i64, deserialize_string_to_i64};
 
 use super::bucket_priority::BucketPriority;
@@ -123,7 +115,8 @@ pub struct OplogEntry<'a> {
 
 #[derive(Debug)]
 pub enum OplogData<'a> {
-    JsonString { data: Cow<'a, str> },
+    /// A string encoding a well-formed JSON object representing values of the row.
+    Json { data: Cow<'a, str> },
     //    BsonDocument { data: Cow<'a, [u8]> },
 }
 
@@ -154,147 +147,12 @@ impl<'a, 'de: 'a> Deserialize<'de> for OplogData<'a> {
     where
         D: serde::Deserializer<'de>,
     {
-        struct ReadFromBsonVisitor;
-
-        impl<'de> Visitor<'de> for ReadFromBsonVisitor {
-            type Value = OplogData<'de>;
-
-            fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result {
-                formatter.write_str("a string or an object")
-            }
-
-            fn visit_borrowed_str<E>(self, v: &'de str) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                // Sync service sent data as JSON string. We will save that same string into
-                // ps_oplog without any transformations.
-                Ok(OplogData::JsonString {
-                    data: Cow::Borrowed(v),
-                })
-            }
-
-            fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-            where
-                E: serde::de::Error,
-            {
-                // Same case, but if the deserializer doesn't let us borrow the JSON string.
-                Ok(OplogData::JsonString {
-                    data: Cow::Owned(v.to_string()),
-                })
-            }
-
-            fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
-            where
-                A: serde::de::MapAccess<'de>,
-            {
-                // Ok, we have a sub-document / JSON object. We can't save that as-is, we need to
-                // serialize it. serde_json's Serializer is std-only because they don't want to
-                // expose their custom no_std Write trait. So we have to use our own writer impl
-                // here.
-
-                let mut writer = JsonWriter::new();
-
-                struct PendingKey<'a, 'de> {
-                    key: &'de str,
-                    writer: &'a mut JsonWriter,
-                }
-
-                impl<'a, 'de> Visitor<'de> for PendingKey<'a, 'de> {
-                    type Value = ();
-
-                    fn expecting(&self, formatter: &mut core::fmt::Formatter) -> core::fmt::Result {
-                        formatter.write_str("SQLite-compatible value")
-                    }
-
-                    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-                    where
-                        E: serde::de::Error,
-                    {
-                        self.writer.write_str(self.key, v);
-                        Ok(())
-                    }
-
-                    fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E>
-                    where
-                        E: serde::de::Error,
-                    {
-                        self.writer.write_f64(self.key, v);
-                        Ok(())
-                    }
-
-                    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
-                    where
-                        E: serde::de::Error,
-                    {
-                        self.writer.write_i64(self.key, v as i64);
-                        Ok(())
-                    }
-
-                    fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
-                    where
-                        E: serde::de::Error,
-                    {
-                        self.writer.write_i64(self.key, v);
-                        Ok(())
-                    }
-                }
-
-                impl<'a, 'de> DeserializeSeed<'de> for PendingKey<'a, 'de> {
-                    type Value = ();
-
-                    fn deserialize<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
-                    where
-                        D: serde::Deserializer<'de>,
-                    {
-                        deserializer.deserialize_any(self)
-                    }
-                }
-
-                while let Some(key) = map.next_key::<&'de str>()? {
-                    let pending = PendingKey {
-                        key,
-                        writer: &mut writer,
-                    };
-                    map.next_value_seed(pending)?;
-                }
-
-                Ok(OplogData::JsonString {
-                    data: Cow::Owned(writer.finish()),
-                })
-            }
-        }
-
-        // Regardless of whether we're deserializing JSON or BSON, oplog data is represented either
-        // as a string (representing a JSON-encoded object) or an object (representing the values
-        // directly).
-
-        let is_from_bson = !deserializer.is_human_readable();
-        if is_from_bson {
-            deserializer.deserialize_any(ReadFromBsonVisitor)
-        } else {
-            // We're already coming from JSON, so we either have a JSON string or a JSON object.
-            // Let's take a look at the serialized JSON string.
-            let data: &'de RawValue = Deserialize::deserialize(deserializer)?;
-            let str = data.get();
-
-            if matches!(str.chars().nth(0), Some('"')) {
-                // We have a JSON object serialized into a string. We'll have to deserialize once
-                // so that we have the JSON form of the object itself to forward to the database.
-                // This turns `"{\"foo\"": 1}"` into `{"foo": 1}`
-                let content: String = serde_json::from_str(str)
-                    .map_err(|_| D::Error::custom("could not deserialize json string"))?;
-                Ok(OplogData::JsonString {
-                    data: content.into(),
-                })
-            } else {
-                debug_assert_matches!(str.chars().nth(0), Some('{'));
-
-                // It's an embedded object that we now have as a string. How convenient, we'll save
-                // that into the database without further modifications.
-                Ok(OplogData::JsonString { data: str.into() })
-            }
-        }
+        // For now, we will always get oplog data as a string. In the future, there may be the
+        // option of the sync service sending BSON-encoded data lines too, but that's not relevant
+        // for now.
+        return Ok(OplogData::Json {
+            data: Deserialize::deserialize(deserializer)?,
+        });
     }
 }
 

crates/core/src/sync/line.rs

@@ -136,7 +136,7 @@ INSERT OR IGNORE INTO ps_updated_rows(row_type, row_id) VALUES(?1, ?2)",
                 }
 
                 match data.data {
-                    Some(OplogData::JsonString { ref data }) => {
+                    Some(OplogData::Json { ref data }) => {
                         insert_statement.bind_text(6, data.as_ref(), sqlite::Destructor::STATIC)?
                     }
                     // Some(OplogData::BsonDocument { ref data }) => {

crates/core/src/sync/operations.rs

@@ -87,9 +87,7 @@
             "object_type": "items",
             "object_id": "1",
             "checksum": 0,
-            "data": {
-              "col": "hi"
-            }
+            "data": "{\"col\":\"hi\"}"
           }
         ]
       }

dart/test/goldens/simple_iteration.json

@@ -43,7 +43,7 @@ void main() {
                 'object_id': '1',
                 'subkey': json.encode('subkey'),
                 'checksum': 0,
-                'data': {'col': 'a'},
+                'data': json.encode({'col': 'a'}),
               }
             ],
           }

dart/test/js_key_encoding_test.dart

@@ -53,7 +53,7 @@ void main() {
                 'object_type': 'items',
                 'object_id': rowId,
                 'checksum': 0,
-                'data': data,
+                'data': json.encode(data),
               }
             ],
           }

dart/test/legacy_sync_test.dart

@@ -106,7 +106,7 @@ void _syncTests<T>({
             'object_type': 'items',
             'object_id': rowId,
             'checksum': checksum,
-            'data': data,
+            'data': json.encode(data),
           }
         ],
       },