feat(S3): added S3ObjectPath

mcakircali · mcakircali · commit 14f357944393 · 2024-12-19T22:42:12.000+01:00
diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt
@@ -305,6 +305,8 @@ if( eckit_HAVE_AWS_S3 )
         io/s3/S3Name.h
         io/s3/S3ObjectName.cc
         io/s3/S3ObjectName.h
+        io/s3/S3ObjectPath.cc
+        io/s3/S3ObjectPath.h
         io/s3/S3Session.cc
         io/s3/S3Session.h
         io/s3/S3URIManager.cc
diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc
@@ -21,6 +21,7 @@
 #include "eckit/io/s3/S3Exception.h"
 #include "eckit/io/s3/S3Name.h"
 #include "eckit/io/s3/S3ObjectName.h"
+#include "eckit/io/s3/S3ObjectPath.h"
 #include "eckit/log/Log.h"
 #include "eckit/net/Endpoint.h"
 
@@ -34,15 +35,19 @@ namespace eckit {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-S3BucketName::S3BucketName(const URI& uri) : S3Name(uri) {
-    const auto pairs = parse(uri.name());
-    if (pairs.empty()) { throw S3SeriousBug("Could not parse bucket name!", Here()); }
-    bucket_ = pairs[0];
+auto S3BucketName::parse(const std::string& name) -> std::string {
+    const auto parsed = S3Name::parse(name);
+    if (parsed.size() != 1) { throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); }
+    return {parsed[0]};
 }
 
+//----------------------------------------------------------------------------------------------------------------------
+
 S3BucketName::S3BucketName(const net::Endpoint& endpoint, std::string bucket)
     : S3Name(endpoint), bucket_ {std::move(bucket)} { }
 
+S3BucketName::S3BucketName(const URI& uri) : S3Name(uri), bucket_ {parse(uri.name())} { }
+
 //----------------------------------------------------------------------------------------------------------------------
 
 auto S3BucketName::uri() const -> URI {
@@ -63,7 +68,7 @@ void S3BucketName::print(std::ostream& out) const {
 //----------------------------------------------------------------------------------------------------------------------
 
 auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr<S3ObjectName> {
-    return std::make_unique<S3ObjectName>(endpoint(), bucket_, object);
+    return std::make_unique<S3ObjectName>(endpoint(), S3ObjectPath {bucket_, object});
 }
 
 auto S3BucketName::exists() const -> bool {
diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h
@@ -35,6 +35,9 @@ class S3ObjectName;
 //----------------------------------------------------------------------------------------------------------------------
 
 class S3BucketName : public S3Name {
+public:  // methods
+    static auto parse(const std::string& name) -> std::string;
+
 public:  // methods
     explicit S3BucketName(const URI& uri);
 
diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h
@@ -20,6 +20,7 @@
 #pragma once
 
 #include "eckit/io/s3/S3Config.h"
+#include "eckit/io/s3/S3ObjectPath.h"
 
 #include <cstdint>
 #include <iosfwd>
@@ -47,6 +48,8 @@ class S3Client {
 
     auto config() const -> const S3Config& { return config_; }
 
+    // bucket operations
+
     virtual void createBucket(const std::string& bucket) const = 0;
 
     virtual void emptyBucket(const std::string& bucket) const = 0;
@@ -57,26 +60,21 @@ class S3Client {
 
     virtual auto listBuckets() const -> std::vector<std::string> = 0;
 
-    virtual auto putObject(const std::string& bucket,
-                           const std::string& object,
-                           const void*        buffer,
-                           uint64_t           length) const -> long long = 0;
+    // object operations
+
+    virtual auto putObject(const S3ObjectPath& path, const void* buffer, uint64_t length) const -> long long = 0;
 
-    virtual auto getObject(const std::string& bucket,
-                           const std::string& object,
-                           void*              buffer,
-                           uint64_t           offset,
-                           uint64_t           length) const -> long long = 0;
+    virtual auto getObject(const S3ObjectPath& path, void* buffer, uint64_t offset, uint64_t length) const -> long long = 0;
 
-    virtual void deleteObject(const std::string& bucket, const std::string& object) const = 0;
+    virtual void deleteObject(const S3ObjectPath& path) const = 0;
 
     virtual void deleteObjects(const std::string& bucket, const std::vector<std::string>& objects) const = 0;
 
     virtual auto listObjects(const std::string& bucket) const -> std::vector<std::string> = 0;
 
-    virtual auto objectExists(const std::string& bucket, const std::string& object) const -> bool = 0;
+    virtual auto objectExists(const S3ObjectPath& path) const -> bool = 0;
 
-    virtual auto objectSize(const std::string& bucket, const std::string& object) const -> long long = 0;
+    virtual auto objectSize(const S3ObjectPath& path) const -> long long = 0;
 
 protected:  // methods
     S3Client();
diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc
@@ -17,7 +17,10 @@
 
 #include "eckit/exception/Exceptions.h"
 #include "eckit/filesystem/URI.h"
+#include "eckit/io/s3/S3BucketName.h"
 #include "eckit/io/s3/S3Client.h"
+#include "eckit/io/s3/S3ObjectName.h"
+#include "eckit/io/s3/S3ObjectPath.h"
 #include "eckit/io/s3/S3Session.h"
 #include "eckit/utils/Tokenizer.h"
 
@@ -34,11 +37,20 @@ auto S3Name::parse(const std::string& name) -> std::vector<std::string> {
     return Tokenizer("/").tokenize(name);
 }
 
+auto S3Name::make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr<S3Name> {
+    const auto names = parse(name);
+    switch (names.size()) {
+        case 1:  return std::make_unique<S3BucketName>(endpoint, names[0]);
+        case 2:  return std::make_unique<S3ObjectName>(endpoint, S3ObjectPath {names[0], names[1]});
+        default: throw SeriousBug("Could not parse S3 name: " + name, Here());
+    }
+}
+
 //----------------------------------------------------------------------------------------------------------------------
 
 S3Name::S3Name(const net::Endpoint& endpoint) : endpoint_ {endpoint} { }
 
-S3Name::S3Name(const URI& uri) : S3Name( {uri.host(), uri.port()}) {
+S3Name::S3Name(const URI& uri) : endpoint_ {uri.host(), uri.port()} {
     /// @todo is "s3://endpoint/bucket/object" a valid URI ?
     ASSERT(uri.scheme() == type);
 }
@@ -62,8 +74,7 @@ void S3Name::print(std::ostream& out) const {
 //----------------------------------------------------------------------------------------------------------------------
 
 auto S3Name::client() const -> S3Client& {
-    if (!client_) { client_ = S3Session::instance().getClient(endpoint_); }
-    return *client_;
+    return *S3Session::instance().getClient(endpoint_);
 }
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h
@@ -34,11 +34,13 @@ class S3Client;
 //----------------------------------------------------------------------------------------------------------------------
 
 class S3Name {
-public:  // types
+public:  // statics
     static constexpr auto type = "s3";
 
     static auto parse(const std::string& name) -> std::vector<std::string>;
 
+    static auto make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr<S3Name>;
+
 public:  // methods
     explicit S3Name(const net::Endpoint& endpoint);
 
@@ -56,6 +58,8 @@ class S3Name {
 
     // accessors
 
+    void endpoint(const net::Endpoint& endpoint) { endpoint_ = endpoint; }
+
     auto endpoint() const -> const net::Endpoint& { return endpoint_; }
 
     virtual auto uri() const -> URI;
@@ -73,8 +77,6 @@ class S3Name {
 
 private:  // members
     net::Endpoint endpoint_;
-
-    mutable std::shared_ptr<S3Client> client_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc
@@ -20,6 +20,7 @@
 #include "eckit/io/s3/S3Exception.h"
 #include "eckit/io/s3/S3Handle.h"
 #include "eckit/io/s3/S3Name.h"
+#include "eckit/io/s3/S3ObjectPath.h"
 #include "eckit/log/CodeLocation.h"
 #include "eckit/net/Endpoint.h"
 
@@ -31,57 +32,60 @@ namespace eckit {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri) {
-    const auto pairs = parse(uri.name());
-    if (pairs.size() != 2) { throw S3SeriousBug("Could not parse bucket and object names!", Here()); }
-    bucket_ = pairs[0];
-    object_ = pairs[1];
+auto S3ObjectName::parse(const std::string& name) -> S3ObjectPath {
+    const auto parsed = S3Name::parse(name);
+    if (parsed.size() != 2) { throw S3SeriousBug("Could not parse bucket/object from name: " + name, Here()); }
+    return {parsed[0], parsed[1]};
 }
 
-S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object)
-    : S3Name(endpoint), bucket_ {std::move(bucket)}, object_ {std::move(object)} { }
+//----------------------------------------------------------------------------------------------------------------------
+
+S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path)
+    : S3Name(endpoint), path_ {std::move(path)} { }
+
+S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri), path_ {parse(uri.name())} { }
 
 //----------------------------------------------------------------------------------------------------------------------
 
 void S3ObjectName::print(std::ostream& out) const {
-    out << "S3ObjectName[object=" << object_ << ",bucket=" << bucket_;
+    out << "S3ObjectName[path=" << path_;
     S3Name::print(out);
 }
 
 //----------------------------------------------------------------------------------------------------------------------
 
 auto S3ObjectName::uri() const -> URI {
     auto uri = S3Name::uri();
-    uri.path("/" + bucket_ + "/" + object_);
+    uri.path(path_);
     return uri;
 }
 
 auto S3ObjectName::asString() const -> std::string {
-    return S3Name::asString() + "/" + bucket_ + "/" + object_;
+    return S3Name::asString() + '/' + path_.asString();
 }
 
 auto S3ObjectName::size() const -> long long {
-    return client().objectSize(bucket_, object_);
+    return client().objectSize(path_);
 }
 
 auto S3ObjectName::exists() const -> bool {
-    return client().objectExists(bucket_, object_);
+    return client().objectExists(path_);
 }
 
 auto S3ObjectName::bucketExists() const -> bool {
-    return client().bucketExists(bucket_);
+    return client().bucketExists(path_.bucket);
 }
 
 void S3ObjectName::remove() {
-    client().deleteObject(bucket_, object_);
+    client().deleteObject(path_);
 }
 
 auto S3ObjectName::put(const void* buffer, const long length) const -> long long {
-    return client().putObject(bucket_, object_, buffer, length);
+    return client().putObject(path_, buffer, length);
 }
 
 auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long {
-    return client().getObject(bucket_, object_, buffer, offset, length);
+    return client().getObject(path_, buffer, offset, length);
 }
 
 auto S3ObjectName::dataHandle() -> DataHandle* {
diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h
@@ -20,7 +20,7 @@
 #pragma once
 
 #include "eckit/io/s3/S3Name.h"
-#include "eckit/net/Endpoint.h"
+#include "eckit/io/s3/S3ObjectPath.h"
 
 #include <ostream>
 #include <string>
@@ -33,10 +33,13 @@ class DataHandle;
 //----------------------------------------------------------------------------------------------------------------------
 
 class S3ObjectName : public S3Name {
+public:  // helpers
+    static auto parse(const std::string& name) -> S3ObjectPath;
+
 public:  // methods
-    explicit S3ObjectName(const URI& uri);
+    S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path);
 
-    S3ObjectName(const net::Endpoint& endpoint, std::string bucket, std::string object);
+    explicit S3ObjectName(const URI& uri);
 
     auto uri() const -> URI override;
 
@@ -58,18 +61,19 @@ class S3ObjectName : public S3Name {
 
     auto asString() const -> std::string override;
 
-    auto name() const -> const std::string& { return object_; }
+    auto path() const -> const S3ObjectPath& { return path_; }
+
+    auto name() const -> const std::string& { return path_.object; }
 
-    auto bucket() const -> const std::string& { return bucket_; }
+    auto bucket() const -> const std::string& { return path_.bucket; }
 
     auto bucketExists() const -> bool;
 
 private:  // methods
     void print(std::ostream& out) const override;
 
 private:  // members
-    std::string bucket_;
-    std::string object_;
+    S3ObjectPath path_;
 };
 
 //----------------------------------------------------------------------------------------------------------------------
diff --git a/src/eckit/io/s3/S3ObjectPath.cc b/src/eckit/io/s3/S3ObjectPath.cc
@@ -0,0 +1,32 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project IO-SEA
+ * (Project ID: 955811) iosea-project.eu
+ */
+
+#include "eckit/io/s3/S3ObjectPath.h"
+
+#include <ostream>
+
+namespace eckit {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) {
+    out << "object=" << path.object << ", bucket=" << path.bucket;
+    return out;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit
+
diff --git a/src/eckit/io/s3/S3ObjectPath.h b/src/eckit/io/s3/S3ObjectPath.h
@@ -0,0 +1,40 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project IO-SEA
+ * (Project ID: 955811) iosea-project.eu
+ */
+
+/// @file   S3ObjectPath.h
+/// @author Metin Cakircali
+/// @date   Dec 2024
+
+#pragma once
+
+#include <ostream>
+#include <string>
+
+namespace eckit {
+
+struct S3ObjectPath {
+    std::string bucket;
+    std::string object;
+
+    auto asString() const -> std::string { return bucket + '/' + object; }
+
+    operator std::string() const { return asString(); }
+
+    friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path);
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace eckit
diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc
diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h
