feat: Add time_expires as first-class metadata (#217)

Author: lcian

clients/python/src/objectstore_client/metadata.py

@@ -11,6 +11,7 @@
 
 HEADER_EXPIRATION = "x-sn-expiration"
 HEADER_TIME_CREATED = "x-sn-time-created"
+HEADER_TIME_EXPIRES = "x-sn-time-expires"
 HEADER_META_PREFIX = "x-snme-"
 
 
@@ -35,10 +36,22 @@ class Metadata:
     time_created: datetime | None
     """
     Timestamp indicating when the object was created or the last time it was replaced.
+
     This means that a PUT request to an existing object causes this value to be bumped.
     This field is computed by the server, it cannot be set by clients.
     """
 
+    time_expires: datetime | None
+    """
+    Timestamp indicating when the object will expire.
+
+    When using a Time To Idle expiration policy, this value will reflect the expiration
+    timestamp present prior to the current access to the object.
+
+    This field is computed by the server, it cannot be set by clients.
+    Use `expiration_policy` to set an expiration policy instead.
+    """
+
     custom: dict[str, str]
 
     @classmethod
@@ -47,6 +60,7 @@ def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
         compression = None
         expiration_policy = None
         time_created = None
+        time_expires = None
         custom_metadata = {}
 
         for k, v in headers.items():
@@ -58,6 +72,8 @@ def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
                 expiration_policy = parse_expiration(v)
             elif k == HEADER_TIME_CREATED:
                 time_created = datetime.fromisoformat(v)
+            elif k == HEADER_TIME_EXPIRES:
+                time_expires = datetime.fromisoformat(v)
             elif k.startswith(HEADER_META_PREFIX):
                 custom_metadata[k[len(HEADER_META_PREFIX) :]] = v
 
@@ -66,6 +82,7 @@ def from_headers(cls, headers: Mapping[str, str]) -> Metadata:
             compression=compression,
             expiration_policy=expiration_policy,
             time_created=time_created,
+            time_expires=time_expires,
             custom=custom_metadata,
         )
 

objectstore-service/src/backend/bigtable.rs

@@ -263,6 +263,7 @@ impl Backend for BigTableBackend {
         // TODO: Inject the access time from the request.
         let access_time = SystemTime::now();
         metadata.size = Some(value.len());
+        metadata.time_expires = expire_at;
 
         // Filter already expired objects but leave them to garbage collection
         if metadata.expiration_policy.is_timeout() && expire_at.is_some_and(|ts| ts < access_time) {

objectstore-service/src/backend/gcs.rs

@@ -144,6 +144,7 @@ impl GcsObject {
             size,
             custom,
             time_created,
+            time_expires: self.custom_time,
         })
     }
 }
@@ -478,6 +479,7 @@ mod tests {
             compression: None,
             custom: BTreeMap::from_iter([("hello".into(), "world".into())]),
             time_created: Some(SystemTime::now()),
+            time_expires: None,
             size: None,
         };
 

objectstore-service/src/backend/local_fs.rs

@@ -130,6 +130,7 @@ mod tests {
             content_type: "text/plain".into(),
             expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_secs(3600)),
             time_created: Some(SystemTime::now()),
+            time_expires: None,
             compression: Some(Compression::Zstd),
             custom: [("foo".into(), "bar".into())].into(),
             size: None,

objectstore-types/src/lib.rs

@@ -24,6 +24,8 @@ pub const HEADER_EXPIRATION: &str = "x-sn-expiration";
 pub const HEADER_REDIRECT_TOMBSTONE: &str = "x-sn-redirect-tombstone";
 /// The custom HTTP header that contains the object creation time.
 pub const HEADER_TIME_CREATED: &str = "x-sn-time-created";
+/// The custom HTTP header that contains the object expiration time.
+pub const HEADER_TIME_EXPIRES: &str = "x-sn-time-expires";
 /// The prefix for custom HTTP headers containing custom per-object metadata.
 pub const HEADER_META_PREFIX: &str = "x-snme-";
 
@@ -204,6 +206,13 @@ pub struct Metadata {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub time_created: Option<SystemTime>,
 
+    /// The expiration time of the object, if any, in accordance with its expiration policy.
+    ///
+    /// When using a Time To Idle expiration policy, this value will reflect the expiration
+    /// timestamp present prior to the current access to the object.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub time_expires: Option<SystemTime>,
+
     /// The content type of the object, if known.
     pub content_type: Cow<'static, str>,
 
@@ -261,6 +270,11 @@ impl Metadata {
                             let time = parse_rfc3339(timestamp)?;
                             metadata.time_created = Some(time);
                         }
+                        HEADER_TIME_EXPIRES => {
+                            let timestamp = value.to_str()?;
+                            let time = parse_rfc3339(timestamp)?;
+                            metadata.time_expires = Some(time);
+                        }
                         _ => {
                             // customer-provided metadata
                             if let Some(name) = name.strip_prefix(HEADER_META_PREFIX) {
@@ -288,6 +302,7 @@ impl Metadata {
             compression,
             expiration_policy,
             time_created,
+            time_expires,
             size: _,
             custom,
         } = self;
@@ -319,6 +334,11 @@ impl Metadata {
             let timestamp = format_rfc3339_micros(*time);
             headers.append(name, timestamp.to_string().parse()?);
         }
+        if let Some(time) = time_expires {
+            let name = HeaderName::try_from(format!("{prefix}{HEADER_TIME_EXPIRES}"))?;
+            let timestamp = format_rfc3339_micros(*time);
+            headers.append(name, timestamp.to_string().parse()?);
+        }
 
         // customer-provided metadata
         for (key, value) in custom {
@@ -343,6 +363,7 @@ impl Default for Metadata {
             is_redirect_tombstone: None,
             expiration_policy: ExpirationPolicy::Manual,
             time_created: None,
+            time_expires: None,
             content_type: DEFAULT_CONTENT_TYPE.into(),
             compression: None,
             size: None,