feat: Add time_creation as first-class metadata (#210)

Author: lcian

clients/python/src/objectstore_client/metadata.py

@@ -4,12 +4,13 @@
 import re
 from collections.abc import Iterable, Iterator, Mapping
 from dataclasses import dataclass
-from datetime import timedelta
+from datetime import datetime, timedelta
 from typing import Literal, TypeVar, cast
 
 Compression = Literal["zstd"] | Literal["none"]
 
 HEADER_EXPIRATION = "x-sn-expiration"
+HEADER_TIME_CREATED = "x-sn-time-created"
 HEADER_META_PREFIX = "x-snme-"
 
 
@@ -31,13 +32,21 @@ class Metadata:
     content_type: str | None
     compression: Compression | None
     expiration_policy: ExpirationPolicy | None
+    time_created: datetime | None
+    """
+    Timestamp indicating when the object was created or the last time it was replaced.
+    This means that a PUT request to an existing object causes this value to be bumped.
+    This field is computed by the server, it cannot be set by clients.
+    """
+
     custom: dict[str, str]
 
     @classmethod
     def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
         content_type = "application/octet-stream"
         compression = None
         expiration_policy = None
+        time_created = None
         custom_metadata = {}
 
         for k, v in headers.items():
@@ -47,13 +56,16 @@ def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
                 compression = cast(Compression | None, v)
             elif k == HEADER_EXPIRATION:
                 expiration_policy = parse_expiration(v)
+            elif k == HEADER_TIME_CREATED:
+                time_created = datetime.fromisoformat(v)
             elif k.startswith(HEADER_META_PREFIX):
                 custom_metadata[k[len(HEADER_META_PREFIX) :]] = v
 
         return Metadata(
             content_type=content_type,
             compression=compression,
             expiration_policy=expiration_policy,
+            time_created=time_created,
             custom=custom_metadata,
         )
 

clients/python/tests/test_e2e.py

@@ -110,6 +110,7 @@ def test_full_cycle(server_url: str) -> None:
 
     retrieved = session.get(object_key)
     assert retrieved.payload.read() == data
+    assert retrieved.metadata.time_created is not None
 
     session.delete(object_key)
 

clients/rust/src/tests.rs

@@ -34,6 +34,7 @@ async fn stores_uncompressed() {
 
     assert_eq!(metadata.compression, None);
     assert_eq!(received.as_ref(), b"oh hai!");
+    assert!(metadata.time_created.is_some());
 }
 
 #[tokio::test]

objectstore-server/src/endpoints.rs

@@ -1,6 +1,7 @@
 //! Contains all HTTP endpoint handlers.
 
 use std::io;
+use std::time::SystemTime;
 
 use anyhow::Context;
 use axum::body::Body;
@@ -44,8 +45,10 @@ async fn put_object(
     body: Body,
 ) -> ApiResult<impl IntoResponse> {
     populate_sentry_scope(&path);
-    let metadata =
+
+    let mut metadata =
         Metadata::from_headers(&headers, "").context("extracting metadata from headers")?;
+    metadata.time_created = Some(SystemTime::now());
 
     let stream = body.into_data_stream().map_err(io::Error::other).boxed();
     let key = state.service.put_object(path, &metadata, stream).await?;

objectstore-service/src/backend/bigtable.rs

@@ -371,6 +371,7 @@ mod tests {
         let path = make_key();
         let metadata = Metadata {
             content_type: "text/plain".into(),
+            time_created: Some(SystemTime::now()),
             custom: BTreeMap::from_iter([("hello".into(), "world".into())]),
             ..Default::default()
         };

objectstore-service/src/backend/gcs.rs

@@ -55,6 +55,14 @@ struct GcsObject {
     /// without having to stream it.
     pub size: Option<String>,
 
+    /// Timestamp of when this object was created.
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        with = "humantime_serde"
+    )]
+    pub time_created: Option<SystemTime>,
+
     /// User-provided metadata, including our built-in metadata.
     #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub metadata: BTreeMap<GcsMetaKey, String>,
@@ -68,6 +76,7 @@ impl GcsObject {
             size: metadata.size.map(|size| size.to_string()),
             content_encoding: None,
             custom_time: None,
+            time_created: metadata.time_created,
             metadata: BTreeMap::new(),
         };
 
@@ -113,6 +122,7 @@ impl GcsObject {
         let content_type = self.content_type;
         let compression = self.content_encoding.map(|s| s.parse()).transpose()?;
         let size = self.size.map(|size| size.parse()).transpose()?;
+        let time_created = self.time_created;
 
         // At this point, all built-in metadata should have been removed from self.metadata.
         let mut custom = BTreeMap::new();
@@ -131,8 +141,9 @@ impl GcsObject {
             content_type,
             expiration_policy,
             compression,
-            custom,
             size,
+            custom,
+            time_created,
         })
     }
 }
@@ -466,6 +477,7 @@ mod tests {
             expiration_policy: ExpirationPolicy::Manual,
             compression: None,
             custom: BTreeMap::from_iter([("hello".into(), "world".into())]),
+            time_created: Some(SystemTime::now()),
             size: None,
         };
 
@@ -480,6 +492,7 @@ mod tests {
         assert_eq!(str_payload, "hello, world");
         assert_eq!(meta.content_type, metadata.content_type);
         assert_eq!(meta.custom, metadata.custom);
+        assert!(metadata.time_created.is_some());
 
         Ok(())
     }

objectstore-service/src/backend/local_fs.rs

@@ -103,7 +103,7 @@ impl Backend for LocalFsBackend {
 
 #[cfg(test)]
 mod tests {
-    use std::time::Duration;
+    use std::time::{Duration, SystemTime};
 
     use bytes::BytesMut;
     use futures_util::TryStreamExt;
@@ -129,6 +129,7 @@ mod tests {
             is_redirect_tombstone: None,
             content_type: "text/plain".into(),
             expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_secs(3600)),
+            time_created: Some(SystemTime::now()),
             compression: Some(Compression::Zstd),
             custom: [("foo".into(), "bar".into())].into(),
             size: None,

objectstore-types/src/lib.rs

@@ -13,13 +13,17 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use http::header::{self, HeaderMap, HeaderName};
-use humantime::{format_duration, format_rfc3339_seconds, parse_duration};
+use humantime::{
+    format_duration, format_rfc3339_micros, format_rfc3339_seconds, parse_duration, parse_rfc3339,
+};
 use serde::{Deserialize, Serialize};
 
 /// The custom HTTP header that contains the serialized [`ExpirationPolicy`].
 pub const HEADER_EXPIRATION: &str = "x-sn-expiration";
 /// The custom HTTP header that contains the serialized redirect tombstone.
 pub const HEADER_REDIRECT_TOMBSTONE: &str = "x-sn-redirect-tombstone";
+/// The custom HTTP header that contains the object creation time.
+pub const HEADER_TIME_CREATED: &str = "x-sn-time-created";
 /// The prefix for custom HTTP headers containing custom per-object metadata.
 pub const HEADER_META_PREFIX: &str = "x-snme-";
 
@@ -41,6 +45,9 @@ pub enum Error {
     /// The content type is invalid.
     #[error("invalid content type")]
     InvalidContentType(#[from] mediatype::MediaTypeError),
+    /// The creation time is invalid.
+    #[error("invalid creation time")]
+    InvalidCreationTime(#[from] humantime::TimestampError),
 }
 impl From<http::header::InvalidHeaderValue> for Error {
     fn from(err: http::header::InvalidHeaderValue) -> Self {
@@ -190,6 +197,13 @@ pub struct Metadata {
     #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")]
     pub expiration_policy: ExpirationPolicy,
 
+    /// The creation/last replacement time of the object, if known.
+    ///
+    /// This is populated by the server when performing a POST or PUT request, i.e. when an object is
+    /// first created or when an existing object is overwritten.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub time_created: Option<SystemTime>,
+
     /// The content type of the object, if known.
     pub content_type: Cow<'static, str>,
 
@@ -242,6 +256,11 @@ impl Metadata {
                                 metadata.is_redirect_tombstone = Some(true);
                             }
                         }
+                        HEADER_TIME_CREATED => {
+                            let timestamp = value.to_str()?;
+                            let time = parse_rfc3339(timestamp)?;
+                            metadata.time_created = Some(time);
+                        }
                         _ => {
                             // customer-provided metadata
                             if let Some(name) = name.strip_prefix(HEADER_META_PREFIX) {
@@ -268,6 +287,7 @@ impl Metadata {
             content_type,
             compression,
             expiration_policy,
+            time_created,
             size: _,
             custom,
         } = self;
@@ -294,6 +314,11 @@ impl Metadata {
                 headers.append("x-goog-custom-time", expires_at.to_string().parse()?);
             }
         }
+        if let Some(time) = time_created {
+            let name = HeaderName::try_from(format!("{prefix}{HEADER_TIME_CREATED}"))?;
+            let timestamp = format_rfc3339_micros(*time);
+            headers.append(name, timestamp.to_string().parse()?);
+        }
 
         // customer-provided metadata
         for (key, value) in custom {
@@ -317,6 +342,7 @@ impl Default for Metadata {
         Self {
             is_redirect_tombstone: None,
             expiration_policy: ExpirationPolicy::Manual,
+            time_created: None,
             content_type: DEFAULT_CONTENT_TYPE.into(),
             compression: None,
             size: None,