Add support for byte translation

core/data/tests/test_byte_translation/input.rs

@@ -0,0 +1,5 @@
+#[typeshare]
+#[serde(rename_all = "camelCase")]
+pub struct Foo {
+    pub bytes: Vec<u8>,
+}

core/data/tests/test_byte_translation/output.go

@@ -0,0 +1,7 @@
+package proto
+
+import "encoding/json"
+
+type Foo struct {
+	Bytes []byte `json:"bytes"`
+}

core/data/tests/test_byte_translation/output.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from pydantic import BaseModel, field_serializer, field_validator
+
+
+class Foo(BaseModel):
+    bytes: bytes
+
+
+    @field_serializer("content")
+    def serialize_data(self, value: bytes) -> list[int]:
+        return list(value)
+
+    @field_validator("content", mode="before")
+    def deserialize_data(cls, value):
+        if isinstance(value, list): 
+            return bytes(value)
+        return value

core/data/tests/test_byte_translation/output.ts

@@ -0,0 +1,18 @@
+export interface Foo {
+	bytes: Uint8Array;
+}
+
+// Reviver code  - required for JSON deserialization
+export function TypeshareReviver(key: string, value: unknown): unknown {
+    return Array.isArray(value) && value.every(Number.isFinite) 
+        ? new Uint8Array(value) 
+        : value;
+}
+
+// Replacer code - required for JSON serialization
+export function TypeshareReplacer(key: string, value: unknown): unknown {
+    if (value instanceof Uint8Array) {
+        return Array.from(value);
+    }
+    return value;
+}

core/data/tests/test_serde_iso8601/output.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pydantic import BaseModel
+
+
+class Foo(BaseModel):
+    time: datetime
+

core/src/language/go.rs

@@ -1,6 +1,6 @@
 use std::io::Write;
 
-use crate::language::SupportedLanguage;
+use crate::language::{get_vec_u8_conversion, SupportedLanguage};
 use crate::parser::ParsedData;
 use crate::rename::RenameExt;
 use crate::rust_types::{RustConst, RustConstExpr, RustItem, RustTypeFormatError, SpecialRustType};
@@ -119,7 +119,14 @@ impl Language for Go {
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
         Ok(match special_ty {
-            SpecialRustType::Vec(rtype) => format!("[]{}", self.format_type(rtype, generic_types)?),
+            SpecialRustType::Vec(rtype) => {
+                // TODO: https://github.com/1Password/typeshare/issues/231
+                if let Some(conversion) = get_vec_u8_conversion(special_ty, self.type_map(), rtype)
+                {
+                    return Ok(conversion);
+                }
+                format!("[]{}", self.format_type(rtype, generic_types)?)
+            }
             SpecialRustType::Array(rtype, len) => {
                 format!("[{}]{}", len, self.format_type(rtype, generic_types)?)
             }
@@ -491,14 +498,12 @@ func ({short_name} {full_name}) MarshalJSON() ([]byte, error) {{
         }
 
         write_comments(w, 1, &field.comments)?;
-
         let type_name = match field.type_override(SupportedLanguage::Go) {
             Some(type_override) => type_override.to_owned(),
             None => self
                 .format_type(&field.ty, generic_types)
                 .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?,
         };
-
         let go_type = self.acronyms_to_uppercase(&type_name);
         let is_optional = field.ty.is_optional() || field.has_default;
         let formatted_renamed_id = format!("{:?}", &field.id.renamed);

core/src/language/mod.rs

@@ -492,6 +492,20 @@ fn used_imports<'a, 'b: 'a>(
     }
     used_imports
 }
+/// this function will acquire the byte translation for the target language that is located inside
+/// the typeshare.toml
+fn get_vec_u8_conversion(
+    special_type: &SpecialRustType,
+    type_mappings: &HashMap<String, String>,
+    inner_type: &RustType,
+) -> Option<String> {
+    let type_key = format!("{}<{}>", special_type.id(), inner_type.id());
+
+    if inner_type.contains_type(SpecialRustType::U8.id()) {
+        return type_mappings.get(&type_key).map(ToString::to_string);
+    }
+    None
+}
 
 #[cfg(test)]
 mod test {

core/src/language/python.rs

@@ -13,7 +13,7 @@ use std::hash::Hash;
 use std::sync::OnceLock;
 use std::{collections::HashMap, io::Write};
 
-use super::CrateTypes;
+use super::{get_vec_u8_conversion, CrateTypes};
 
 use convert_case::{Case, Casing};
 
@@ -137,6 +137,7 @@ impl Language for Python {
         parameters: &[RustType],
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
+        self.add_imports(base);
         if let Some(mapped) = self.type_map().get(base) {
             Ok(mapped.into())
         } else {
@@ -174,9 +175,16 @@ impl Language for Python {
         generic_types: &[String],
     ) -> Result<String, RustTypeFormatError> {
         match special_ty {
-            SpecialRustType::Vec(rtype)
-            | SpecialRustType::Array(rtype, _)
-            | SpecialRustType::Slice(rtype) => {
+            SpecialRustType::Vec(rtype) => {
+                // TODO: https://github.com/1Password/typeshare/issues/231
+                if let Some(conversion) = get_vec_u8_conversion(special_ty, self.type_map(), rtype)
+                {
+                    return Ok(conversion);
+                }
+                self.add_import("typing".to_string(), "List".to_string());
+                Ok(format!("List[{}]", self.format_type(rtype, generic_types)?))
+            }
+            SpecialRustType::Array(rtype, _) | SpecialRustType::Slice(rtype) => {
                 self.add_import("typing".to_string(), "List".to_string());
                 Ok(format!("List[{}]", self.format_type(rtype, generic_types)?))
             }
@@ -267,6 +275,7 @@ impl Language for Python {
     }
 
     fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> {
+        self.add_import("pydantic".to_string(), "BaseModel".to_string());
         {
             rs.generic_types
                 .iter()
@@ -294,8 +303,31 @@ impl Language for Python {
             write!(w, "    pass")?
         }
         writeln!(w)?;
-        self.add_import("pydantic".to_string(), "BaseModel".to_string());
-        Ok(())
+        rs.fields.iter().try_for_each(|field| {
+            let python_type = self
+                .format_type(&field.ty, &rs.generic_types)
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+            // Ideally, we should statically compare this to SpecialRustType::Bytes
+            // This would cause too much of a refactor
+            if python_type == "bytes" {
+                self.add_import("pydantic".to_owned(), "field_validator".to_owned());
+                self.add_import("pydantic".to_owned(), "field_serializer".to_owned());
+                return writeln!(
+                    w,
+                    r#"    
+    @field_serializer("content")
+    def serialize_data(self, value: bytes) -> list[int]:
+        return list(value)
+
+    @field_validator("content", mode="before")
+    def deserialize_data(cls, value):
+        if isinstance(value, list): 
+            return bytes(value)
+        return value"#
+                );
+            }
+            Ok(())
+        })
     }
 
     fn write_enum(&mut self, w: &mut dyn Write, e: &RustEnum) -> std::io::Result<()> {

core/src/language/typescript.rs

@@ -14,7 +14,7 @@ use std::{
     io::{self, Write},
 };
 
-use super::ScopedCrateTypes;
+use super::{get_vec_u8_conversion, ScopedCrateTypes};
 
 /// All information needed to generate Typescript type-code
 #[derive(Default)]
@@ -38,6 +38,11 @@ impl Language for TypeScript {
     ) -> Result<String, RustTypeFormatError> {
         match special_ty {
             SpecialRustType::Vec(rtype) => {
+                // TODO: https://github.com/1Password/typeshare/issues/231
+                if let Some(conversion) = get_vec_u8_conversion(special_ty, self.type_map(), rtype)
+                {
+                    return Ok(conversion);
+                }
                 Ok(format!("{}[]", self.format_type(rtype, generic_types)?))
             }
             SpecialRustType::Array(rtype, len) => {
@@ -153,7 +158,32 @@ impl Language for TypeScript {
             .iter()
             .try_for_each(|f| self.write_field(w, f, rs.generic_types.as_slice()))?;
 
-        writeln!(w, "}}\n")
+        writeln!(w, "}}\n")?;
+        rs.fields.iter().try_for_each(|field| {
+            let typescript_type = self
+                .format_type(&field.ty, &rs.generic_types)
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
+            if typescript_type == "Uint8Array" {
+                return writeln!(
+                    w,
+                    r#"// Reviver code  - required for JSON deserialization
+export function TypeshareReviver(key: string, value: unknown): unknown {{
+    return Array.isArray(value) && value.every(Number.isFinite) 
+        ? new {typescript_type}(value) 
+        : value;
+}}
+
+// Replacer code - required for JSON serialization
+export function TypeshareReplacer(key: string, value: unknown): unknown {{
+    if (value instanceof {typescript_type}) {{
+        return Array.from(value);
+    }}
+    return value;
+}}"#
+                );
+            }
+            Ok(())
+        })
     }
 
     fn write_enum(&mut self, w: &mut dyn Write, e: &RustEnum) -> io::Result<()> {

core/src/parser.rs

@@ -80,6 +80,8 @@ pub enum ParseError {
     RustConstTypeInvalid,
     #[error("the serde flatten attribute is not currently supported")]
     SerdeFlattenNotAllowed,
+    #[error("the binary data decorator isn't allowed on types other than Vec<u8>")]
+    BinaryDataOnTypesOtherThanBytesNotAllowed,
     #[error("IO error: {0}")]
     IOError(String),
 }

core/tests/snapshot_tests.rs

@@ -424,24 +424,36 @@ static SWIFT_MAPPINGS: Lazy<HashMap<String, String>> = Lazy::new(|| {
 });
 
 static TYPESCRIPT_MAPPINGS: Lazy<HashMap<String, String>> = Lazy::new(|| {
-    [("Url", "string"), ("DateTime", "string")]
-        .iter()
-        .map(|(k, v)| (k.to_string(), v.to_string()))
-        .collect()
+    [
+        ("Url", "string"),
+        ("DateTime", "string"),
+        ("Vec<u8>", "Uint8Array"),
+    ]
+    .iter()
+    .map(|(k, v)| (k.to_string(), v.to_string()))
+    .collect()
 });
 
 static GO_MAPPINGS: Lazy<HashMap<String, String>> = Lazy::new(|| {
-    [("Url", "string"), ("DateTime", "string")]
-        .iter()
-        .map(|(k, v)| (k.to_string(), v.to_string()))
-        .collect()
+    [
+        ("Url", "string"),
+        ("DateTime", "string"),
+        ("Vec<u8>", "[]byte"),
+    ]
+    .iter()
+    .map(|(k, v)| (k.to_string(), v.to_string()))
+    .collect()
 });
 
 static PYTHON_MAPPINGS: Lazy<HashMap<String, String>> = Lazy::new(|| {
-    [("Url", "AnyUrl"), ("DateTime", "datetime")]
-        .iter()
-        .map(|(k, v)| (k.to_string(), v.to_string()))
-        .collect()
+    [
+        ("Url", "AnyUrl"),
+        ("DateTime", "datetime"),
+        ("Vec<u8>", "bytes"),
+    ]
+    .iter()
+    .map(|(k, v)| (k.to_string(), v.to_string()))
+    .collect()
 });
 
 tests! {
@@ -586,6 +598,9 @@ tests! {
          go {
             type_mappings: super::GO_MAPPINGS.clone(),
         },
+        python {
+            type_mappings: super::PYTHON_MAPPINGS.clone(),
+        }
     ];
     test_serde_url: [
         swift {
@@ -674,4 +689,18 @@ tests! {
     excluded_by_target_os: [ swift, kotlin, scala, typescript, go,python ] target_os: ["android", "macos"];
     // excluded_by_target_os_full_module: [swift] target_os: "ios";
     serde_rename_references: [ swift, kotlin, scala, typescript, go ];
+    test_byte_translation: [
+    go
+    {
+        type_mappings: super::GO_MAPPINGS.clone(),
+    },
+    python
+    {
+        type_mappings: super::PYTHON_MAPPINGS.clone(),
+    },
+    typescript
+    {
+        type_mappings: super::TYPESCRIPT_MAPPINGS.clone(),
+    }
+    ];
 }