Parse the media_details JSON into concrete instances (#750)

* Parse the `media_details` JSON into concrete instances

* Update the Swift example app to include some media details

* Add integration tests for media details parsing

* Fix swiftlint issues

* Remove a trailing semicolon

Author: crazytonyli

native/swift/Example/Example/ListViewData.swift

@@ -139,7 +139,13 @@ extension PostWithEditContext: ListViewDataConvertable {
 
 extension MediaWithEditContext: ListViewDataConvertable {
     var asListViewData: ListViewData {
-        ListViewData(id: self.slug, title: self.title.raw, subtitle: String(describing: self.mediaDetails), fields: [:])
+        let details = self.mediaDetails.parseAsMimeType(mimeType: self.mimeType)
+        return ListViewData(
+            id: self.slug,
+            title: details.emoji + " " + (URL(string: self.sourceUrl)?.lastPathComponent ?? "<invalid-source-url>"),
+            subtitle: self.title.raw,
+            fields: details.fields
+        )
     }
 }
 
@@ -160,3 +166,43 @@ extension [ListViewDataConvertable] {
         self.map { $0.asListViewData }
     }
 }
+
+private extension Optional<MediaDetailsPayload> {
+    var emoji: String {
+        switch self {
+        case .audio:
+            "🔊"
+        case .image:
+            "🌆"
+        case .video:
+            "🎥"
+        case .document:
+            "📁"
+        case nil:
+            "❓"
+        }
+    }
+
+    var fields: [String: String] {
+        var fields = [String: String]()
+
+        switch self {
+        case let .audio(audio):
+            fields["Duration"] = "\(audio.length) seconds"
+            fields["File size"] = "\(audio.fileSize) bytes"
+        case let .image(image):
+            fields["Size"] = "\(image.width)x\(image.height) pixels"
+            fields["File size"] = "\(image.fileSize) bytes"
+        case let .video(video):
+            fields["Size"] = "\(video.width)x\(video.height) pixels"
+            fields["Duration"] = "\(video.length) seconds"
+            fields["File size"] = "\(video.fileSize) bytes"
+        case let .document(doc):
+            fields["File size"] = "\(doc.fileSize) bytes"
+        case nil:
+            break
+        }
+
+        return fields
+    }
+}

wp_api/Cargo.toml

@@ -32,7 +32,7 @@ roxmltree = { workspace = true }
 rustls = { workspace = true, optional = true }
 scraper = { workspace = true }
 serde = { workspace = true, features = [ "derive", "rc" ] }
-serde_json = { workspace = true }
+serde_json = { workspace = true, features = [ "raw_value" ] }
 serde_repr = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }

wp_api/src/media.rs

@@ -1,5 +1,5 @@
 use crate::{
-    JsonValue, UserId, WpApiParamOrder,
+    UserId, WpApiParamOrder,
     date::WpGmtDateTime,
     impl_as_query_value_for_new_type, impl_as_query_value_from_to_string,
     posts::{
@@ -11,6 +11,8 @@ use crate::{
     },
 };
 use serde::{Deserialize, Serialize};
+use serde_json::value::RawValue;
+use std::sync::Arc;
 use std::{collections::HashMap, num::ParseIntError, str::FromStr};
 use strum_macros::IntoStaticStr;
 use wp_contextual::WpContextual;
@@ -527,7 +529,7 @@ pub struct SparseMedia {
     #[WpContext(edit, embed, view)]
     pub mime_type: Option<String>,
     #[WpContext(edit, embed, view)]
-    pub media_details: Option<JsonValue>,
+    pub media_details: Option<Arc<MediaDetails>>,
     #[serde(rename = "post")]
     #[WpContext(edit, view)]
     #[WpContextualOption]
@@ -539,6 +541,107 @@ pub struct SparseMedia {
     // meta field is omitted for now: https://github.com/Automattic/wordpress-rs/issues/381
 }
 
+#[derive(Debug, Serialize, Deserialize, uniffi::Object)]
+#[serde(transparent)]
+pub struct MediaDetails {
+    payload: Box<RawValue>,
+}
+
+#[uniffi::export]
+impl MediaDetails {
+    pub fn parse_as_mime_type(&self, mime_type: String) -> Option<MediaDetailsPayload> {
+        if mime_type.starts_with("image/") {
+            return serde_json::from_str::<ImageMediaDetails>(self.payload.get())
+                .ok()
+                .map(MediaDetailsPayload::Image);
+        } else if mime_type.starts_with("video/") || mime_type.starts_with("audio/") {
+            // Some audio files may be returned with a video MIME type, so we attempt to parse both.
+
+            if let Ok(details) = serde_json::from_str::<VideoMediaDetails>(self.payload.get()) {
+                return Some(MediaDetailsPayload::Video(details));
+            }
+
+            if let Ok(details) = serde_json::from_str::<AudioMediaDetails>(self.payload.get()) {
+                return Some(MediaDetailsPayload::Audio(details));
+            }
+        }
+
+        serde_json::from_str::<DocumentMediaDetails>(self.payload.get())
+            .ok()
+            .map(MediaDetailsPayload::Document)
+    }
+}
+
+#[derive(Debug, uniffi::Enum)]
+pub enum MediaDetailsPayload {
+    Audio(AudioMediaDetails),
+    Image(ImageMediaDetails),
+    Video(VideoMediaDetails),
+    Document(DocumentMediaDetails),
+}
+
+#[derive(Debug, Serialize, Deserialize, uniffi::Record)]
+pub struct AudioMediaDetails {
+    #[serde(rename = "filesize")]
+    pub file_size: u64,
+    pub length: u64,
+    pub length_formatted: String,
+
+    #[serde(rename = "dataformat")]
+    pub data_format: Option<String>,
+    pub codec: Option<String>,
+    pub sample_rate: Option<u32>,
+    pub channels: Option<u8>,
+    pub bits_per_sample: Option<u8>,
+    pub lossless: Option<bool>,
+    #[serde(rename = "channelmode")]
+    pub channel_mode: Option<String>,
+    pub bitrate: Option<f64>,
+    pub compression_ratio: Option<f64>,
+    #[serde(rename = "fileformat")]
+    pub file_format: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, uniffi::Record)]
+pub struct ImageMediaDetails {
+    #[serde(rename = "filesize")]
+    pub file_size: u64,
+
+    pub width: u32,
+    pub height: u32,
+    pub file: String,
+    pub sizes: Option<HashMap<String, ScaledImageDetails>>,
+}
+
+#[derive(Debug, Serialize, Deserialize, uniffi::Record)]
+pub struct ScaledImageDetails {
+    pub file: String,
+    pub width: u32,
+    pub height: u32,
+    pub source_url: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, uniffi::Record)]
+pub struct VideoMediaDetails {
+    #[serde(rename = "filesize")]
+    pub file_size: u64,
+    pub length: u32,
+    pub width: u32,
+    pub height: u32,
+
+    #[serde(rename = "fileformat")]
+    pub file_format: Option<String>,
+    #[serde(rename = "dataformat")]
+    pub data_format: Option<String>,
+    pub created_timestamp: Option<u64>,
+}
+
+#[derive(Debug, Serialize, Deserialize, uniffi::Record)]
+pub struct DocumentMediaDetails {
+    #[serde(rename = "filesize")]
+    pub file_size: u64,
+}
+
 #[derive(Debug, Serialize, Deserialize, uniffi::Record, WpContextual)]
 pub struct SparseMediaDescription {
     #[WpContext(edit)]
@@ -650,4 +753,60 @@ mod tests {
             "page=11&per_page=22&search=s_q&{after}&{modified_after}&author=111%2C112&author_exclude=211%2C212&{before}&{modified_before}&exclude=1111%2C1112&include=2111%2C2112&offset=11111&order=desc&orderby=slug&parent=44444%2C44445&search_columns=post_content%2Cpost_excerpt&slug=sl_1%2Csl_2&status=inherit%2Cprivate%2Ctrash&parent_exclude=55555%2C55556&media_type=image&mime_type=image%2Fjpeg"
         )
     }
+
+    #[test]
+    fn media_details_round_trip() {
+        let original = r#"
+        {
+            "id": 11,
+            "date": "2025-05-29T03:15:55",
+            "date_gmt": "2025-05-29T03:15:55",
+            "guid": { "rendered": "https://example.com/dummy.docx" },
+            "modified": "2025-05-29T03:15:55",
+            "modified_gmt": "2025-05-29T03:15:55",
+            "slug": "dummy-slug",
+            "status": "dummy-status",
+            "type": "dummy-type",
+            "link": "https://example.com/dummy-link/",
+            "title": { "rendered": "Dummy Title" },
+            "author": 1,
+            "featured_media": 0,
+            "comment_status": "dummy-comment-status",
+            "ping_status": "dummy-ping-status",
+            "template": "dummy-template",
+            "meta": [],
+            "class_list": [
+                "dummy-class-1",
+                "dummy-class-2",
+                "dummy-class-3",
+                "dummy-class-4",
+                "dummy-class-5"
+            ],
+            "description": {
+                "rendered": "<p class=\"dummy-class\"><a href='https://example.com/dummy.docx'>Dummy Link</a></p>\n"
+            },
+            "caption": {
+                "rendered": "<p>Dummy Caption</p>\n"
+            },
+            "alt_text": "Dummy Alt Text",
+            "media_type": "dummy-media-type",
+            "mime_type": "dummy/mime-type",
+            "media_details": {
+                "filesize": 7378,
+                "sizes": {}
+            },
+            "post": null,
+            "source_url": "https://example.com/dummy.docx"
+            }
+        "#;
+
+        let media = serde_json::from_str::<MediaWithViewContext>(original).unwrap();
+        let serialized = serde_json::to_string(&media).unwrap();
+        let json = serde_json::from_str::<serde_json::Value>(serialized.as_str()).unwrap();
+        assert_eq!(json["media_details"]["filesize"], 7378);
+        assert_eq!(
+            json["media_details"]["sizes"],
+            serde_json::Value::Object(serde_json::Map::new())
+        );
+    }
 }

wp_api_integration_tests/src/lib.rs

@@ -55,6 +55,9 @@ pub const POST_ID_555: PostId = PostId(555);
 pub const POST_ID_DRAFT: PostId = PostId(1164);
 pub const POST_ID_INVALID: PostId = PostId(99999999);
 pub const MEDIA_ID_611: MediaId = MediaId(611);
+pub const MEDIA_ID_VIDEO: MediaId = MediaId(1690);
+pub const MEDIA_ID_AUDIO: MediaId = MediaId(821);
+pub const MEDIA_ID_IMAGE: MediaId = MediaId(1692);
 pub const MEDIA_TEST_FILE_PATH: &str = "../test-data/test_media.jpg";
 pub const MEDIA_TEST_FILE_CONTENT_TYPE: &str = "image/jpeg";
 pub const CATEGORY_ID_48: CategoryId = CategoryId(48);

wp_api_integration_tests/src/prelude.rs

@@ -2,13 +2,13 @@ pub use crate::{
     AssertResponse, AssertWpError, CATEGORY_ID_48, CATEGORY_ID_59, CATEGORY_ID_INVALID,
     CLASSIC_EDITOR_PLUGIN_SLUG, COMMENT_ID_INVALID, EmptyAppNotifier, FIRST_COMMENT_ID,
     FIRST_POST_ID, FIRST_USER_EMAIL, FIRST_USER_ID, HELLO_DOLLY_PLUGIN_SLUG, MEDIA_ID_611,
-    MEDIA_TEST_FILE_CONTENT_TYPE, MEDIA_TEST_FILE_PATH, POST_ID_555, POST_ID_DRAFT,
-    POST_ID_INVALID, POST_TEMPLATE_SINGLE_WITH_SIDEBAR, SECOND_COMMENT_ID, SECOND_USER_EMAIL,
-    SECOND_USER_ID, SECOND_USER_SLUG, TAG_ID_100, TAG_ID_INVALID,
-    TEMPLATE_TWENTY_TWENTY_FOUR_SINGLE, THEME_TWENTY_TWENTY_FIVE, THEME_TWENTY_TWENTY_FOUR,
-    THEME_TWENTY_TWENTY_THREE, TestCredentials, USER_ID_INVALID,
-    WP_ORG_PLUGIN_SLUG_CLASSIC_WIDGETS, api_client, api_client_as_author, api_client_as_subscriber,
-    api_client_with_auth_provider,
+    MEDIA_ID_AUDIO, MEDIA_ID_IMAGE, MEDIA_ID_VIDEO, MEDIA_TEST_FILE_CONTENT_TYPE,
+    MEDIA_TEST_FILE_PATH, POST_ID_555, POST_ID_DRAFT, POST_ID_INVALID,
+    POST_TEMPLATE_SINGLE_WITH_SIDEBAR, SECOND_COMMENT_ID, SECOND_USER_EMAIL, SECOND_USER_ID,
+    SECOND_USER_SLUG, TAG_ID_100, TAG_ID_INVALID, TEMPLATE_TWENTY_TWENTY_FOUR_SINGLE,
+    THEME_TWENTY_TWENTY_FIVE, THEME_TWENTY_TWENTY_FOUR, THEME_TWENTY_TWENTY_THREE, TestCredentials,
+    USER_ID_INVALID, WP_ORG_PLUGIN_SLUG_CLASSIC_WIDGETS, api_client, api_client_as_author,
+    api_client_as_subscriber, api_client_with_auth_provider,
     backend::{Backend, RestoreServer},
     mock::{MockExecutor, response_helpers},
     test_site_api_url_resolver, test_site_url, unwrapped_wp_gmt_date_time,

wp_api_integration_tests/tests/test_media_immut.rs

@@ -126,6 +126,8 @@ async fn paginate_list_media_with_edit_context(#[case] params: MediaListParams)
 pub fn list_cases(#[case] params: MediaListParams) {}
 
 mod filter {
+    use wp_api::media::MediaDetailsPayload;
+
     use super::*;
 
     wp_api::generate_sparse_media_field_with_edit_context_test_cases!();
@@ -254,4 +256,52 @@ mod filter {
             .data;
         media.assert_that_instance_fields_nullability_match_provided_fields(fields);
     }
+
+    #[tokio::test]
+    #[parallel]
+    async fn parse_video_media_details() {
+        let media = api_client()
+            .media()
+            .retrieve_with_edit_context(&MEDIA_ID_VIDEO)
+            .await
+            .assert_response()
+            .data;
+
+        assert!(matches!(
+            media.media_details.parse_as_mime_type(media.mime_type),
+            Some(MediaDetailsPayload::Video { .. })
+        ));
+    }
+
+    #[tokio::test]
+    #[parallel]
+    async fn parse_audio_media_details() {
+        let media = api_client()
+            .media()
+            .retrieve_with_edit_context(&MEDIA_ID_AUDIO)
+            .await
+            .assert_response()
+            .data;
+
+        assert!(matches!(
+            media.media_details.parse_as_mime_type(media.mime_type),
+            Some(MediaDetailsPayload::Audio { .. })
+        ));
+    }
+
+    #[tokio::test]
+    #[parallel]
+    async fn parse_image_media_details() {
+        let media = api_client()
+            .media()
+            .retrieve_with_edit_context(&MEDIA_ID_IMAGE)
+            .await
+            .assert_response()
+            .data;
+
+        assert!(matches!(
+            media.media_details.parse_as_mime_type(media.mime_type),
+            Some(MediaDetailsPayload::Image { .. })
+        ));
+    }
 }