Merge pull request #8418 from shirady/get-object-attributes-api

Add S3 GetObjectAttributes API Implementation

Author: nimrod-becker

src/endpoint/s3/ops/index.js

@@ -39,6 +39,7 @@ exports.get_bucket_versions = require('./s3_get_bucket_versions');
 exports.get_bucket_website = require('./s3_get_bucket_website');
 exports.get_object = require('./s3_get_object');
 exports.get_object_acl = require('./s3_get_object_acl');
+exports.get_object_attributes = require('./s3_get_object_attributes');
 exports.get_object_legal_hold = require('./s3_get_object_legal_hold');
 exports.get_object_retention = require('./s3_get_object_retention');
 exports.get_object_tagging = require('./s3_get_object_tagging');

src/endpoint/s3/ops/s3_get_object_attributes.js

@@ -0,0 +1,93 @@
+/* Copyright (C) 2016 NooBaa */
+'use strict';
+
+const dbg = require('../../../util/debug_module')(__filename);
+const s3_utils = require('../s3_utils');
+const http_utils = require('../../../util/http_utils');
+const S3Error = require('../s3_errors').S3Error;
+
+/**
+ * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGETacl.html
+ */
+async function get_object_attributes(req, res) {
+    const version_id = s3_utils.parse_version_id(req.query.versionId);
+    const encryption = s3_utils.parse_encryption(req);
+    const attributes = _parse_attributes(req);
+
+    const params = {
+        bucket: req.params.bucket,
+        key: req.params.key,
+        version_id: version_id,
+        encryption: encryption, // GAP - we don't use it currently
+        md_conditions: http_utils.get_md_conditions(req), // GAP - we don't use it currently in all namespaces (for example - not in NSFS)
+        attributes: attributes,
+    };
+    dbg.log2('params after parsing', params);
+    const reply = await req.object_sdk.get_object_attributes(params);
+    s3_utils.set_response_headers_get_object_attributes(req, res, reply, version_id);
+    return _make_reply_according_to_attributes(reply, attributes);
+}
+
+/**
+ * _parse_attributes parses the header in which the attributes are passed as tring with ',' as separator
+ * and returns array with the the attributes according to the valid attributes list (otherwise it throws an error)
+ * @param {nb.S3Request} req
+ * @returns {string[]}
+ */
+function _parse_attributes(req) {
+    const attributes_str = req.headers['x-amz-object-attributes'];
+    if (!attributes_str) {
+        dbg.error('get_object_attributes: must pass at least one attribute from:',
+            s3_utils.OBJECT_ATTRIBUTES);
+        throw new S3Error(S3Error.InvalidArgument);
+    }
+    const attributes = attributes_str.split(',').map(item => item.trim());
+    const all_valid = attributes.every(item => s3_utils.OBJECT_ATTRIBUTES.includes(item));
+    if (!all_valid) {
+        dbg.error('get_object_attributes: received attributes:', attributes,
+            'at least one of the attributes is not from:', s3_utils.OBJECT_ATTRIBUTES);
+        throw new S3Error(S3Error.InvalidArgument);
+    }
+    return attributes;
+}
+
+/**
+ * _make_reply_according_to_attributes currently the reply is md_object in most of the namespaces
+ * and we return the properties according to the attributes the client asked for
+ * @param {object} reply
+ * @param {object} attributes
+ * @returns {object}
+ */
+function _make_reply_according_to_attributes(reply, attributes) {
+    const reply_without_filter = {
+        ETag: `"${reply.etag}"`,
+        Checksum: reply.checksum,
+        ObjectParts: reply.object_parts,
+        StorageClass: reply.storage_class,
+        ObjectSize: reply.size
+    };
+    const filtered_reply = {
+        GetObjectAttributesOutput: {
+        }
+    };
+    for (const key of attributes) {
+        if (reply_without_filter[key] === undefined) {
+            dbg.warn('Requested for attributes', attributes,
+                'but currently NooBaa does not support these attributes:',
+                s3_utils.OBJECT_ATTRIBUTES_UNSUPPORTED, '(expect namespace s3)');
+        } else {
+            filtered_reply.GetObjectAttributesOutput[key] = reply_without_filter[key];
+        }
+    }
+    return filtered_reply;
+}
+
+module.exports = {
+    handler: get_object_attributes,
+    body: {
+        type: 'empty',
+    },
+    reply: {
+        type: 'xml',
+    },
+};

src/endpoint/s3/s3_bucket_policy_utils.js

@@ -45,6 +45,7 @@ const OP_NAME_TO_ACTION = Object.freeze({
     get_bucket_object_lock: { regular: "s3:GetBucketObjectLockConfiguration" },
     get_bucket: { regular: "s3:ListBucket" },
     get_object_acl: { regular: "s3:GetObjectAcl" },
+    get_object_attributes: { regular: ["s3:GetObject", "s3:GetObjectAttributes"], versioned: ["s3:GetObjectVersion", "s3:GetObjectVersionAttributes"] }, // Notice - special case
     get_object_tagging: { regular: "s3:GetObjectTagging", versioned: "s3:GetObjectVersionTagging" },
     get_object_uploadId: { regular: "s3:ListMultipartUploadParts" },
     get_object_retention: { regular: "s3:GetObjectRetention"},
@@ -139,11 +140,16 @@ async function _is_object_tag_fit(req, predicate, value) {
 async function has_bucket_policy_permission(policy, account, method, arn_path, req) {
     const [allow_statements, deny_statements] = _.partition(policy.Statement, statement => statement.Effect === 'Allow');
 
+    // the case where the permission is an array started in op get_object_attributes
+    const method_arr = Array.isArray(method) ? method : [method];
+
     // look for explicit denies
-    if (await _is_statements_fit(deny_statements, account, method, arn_path, req)) return 'DENY';
+    const res_arr_deny = await is_statement_fit_of_method_array(deny_statements, account, method_arr, arn_path, req);
+    if (res_arr_deny.every(item => item)) return 'DENY';
 
     // look for explicit allows
-    if (await _is_statements_fit(allow_statements, account, method, arn_path, req)) return 'ALLOW';
+    const res_arr_allow = await is_statement_fit_of_method_array(allow_statements, account, method_arr, arn_path, req);
+    if (res_arr_allow.every(item => item)) return 'ALLOW';
 
     // implicit deny
     return 'IMPLICIT_DENY';
@@ -156,6 +162,7 @@ function _is_action_fit(method, statement) {
         dbg.log1('bucket_policy: ', statement.Action ? 'Action' : 'NotAction', ' fit?', action, method);
         if ((action === '*') || (action === 's3:*') || (action === method)) {
             action_fit = true;
+            break;
         }
     }
     return statement.Action ? action_fit : !action_fit;
@@ -170,6 +177,7 @@ function _is_principal_fit(account, statement) {
         dbg.log1('bucket_policy: ', statement.Principal ? 'Principal' : 'NotPrincipal', ' fit?', principal, account);
         if ((principal.unwrap() === '*') || (principal.unwrap() === account)) {
             principal_fit = true;
+            break;
         }
     }
     return statement.Principal ? principal_fit : !principal_fit;
@@ -184,11 +192,17 @@ function _is_resource_fit(arn_path, statement) {
         dbg.log1('bucket_policy: ', statement.Resource ? 'Resource' : 'NotResource', ' fit?', resource_regex, arn_path);
         if (resource_regex.test(arn_path)) {
             resource_fit = true;
+            break;
         }
     }
     return statement.Resource ? resource_fit : !resource_fit;
 }
 
+async function is_statement_fit_of_method_array(statements, account, method_arr, arn_path, req) {
+    return Promise.all(method_arr.map(method_permission =>
+        _is_statements_fit(statements, account, method_permission, arn_path, req)));
+}
+
 async function _is_statements_fit(statements, account, method, arn_path, req) {
     for (const statement of statements) {
         const action_fit = _is_action_fit(method, statement);
@@ -237,7 +251,7 @@ function _parse_condition_keys(condition_statement) {
 }
 
 async function validate_s3_policy(policy, bucket_name, get_account_handler) {
-    const all_op_names = _.compact(_.flatMap(OP_NAME_TO_ACTION, action => [action.regular, action.versioned]));
+    const all_op_names = _.flatten(_.compact(_.flatMap(OP_NAME_TO_ACTION, action => [action.regular, action.versioned])));
     for (const statement of policy.Statement) {
 
         const statement_principal = statement.Principal || statement.NotPrincipal;

src/endpoint/s3/s3_rest.js

@@ -57,6 +57,7 @@ const OBJECT_SUB_RESOURCES = Object.freeze({
     'legal-hold': 'legal_hold',
     'retention': 'retention',
     'select': 'select',
+    'attributes': 'attributes',
 });
 
 let usage_report = new_usage_report();
@@ -296,6 +297,11 @@ async function authorize_anonymous_access(s3_policy, method, arn_path, req) {
     throw new S3Error(S3Error.AccessDenied);
 }
 
+/**
+ * _get_method_from_req parses the permission needed according to the bucket policy
+ * @param {nb.S3Request} req
+ * @returns {string|string[]}
+ */
 function _get_method_from_req(req) {
     const s3_op = s3_bucket_policy_utils.OP_NAME_TO_ACTION[req.op_name];
     if (!s3_op) {

src/endpoint/s3/s3_utils.js

@@ -38,6 +38,9 @@ const base64_regex = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{
 
 const X_NOOBAA_AVAILABLE_STORAGE_CLASSES = 'x-noobaa-available-storage-classes';
 
+const OBJECT_ATTRIBUTES = Object.freeze(['ETag', 'Checksum', 'ObjectParts', 'StorageClass', 'ObjectSize']);
+const OBJECT_ATTRIBUTES_UNSUPPORTED = Object.freeze(['Checksum', 'ObjectParts']);
+
  /**
  * get_default_object_owner returns bucket_owner info if exists
  * else it'll return the default owner
@@ -324,6 +327,28 @@ function set_response_object_md(res, object_md) {
     }
 }
 
+/** set_response_headers_get_object_attributes is based on set_response_object_md
+ * and serves get_object_attributes
+ * @param {nb.S3Request} req
+ * @param {nb.S3Response} res
+ * @param {object} reply
+ * @param {string} version_id 
+ */
+function set_response_headers_get_object_attributes(req, res, reply, version_id) {
+    if (version_id) {
+        res.setHeader('x-amz-version-id', version_id);
+        if (reply.delete_marker) {
+            res.setHeader('x-amz-delete-marker', 'true');
+        }
+    }
+    if (reply.last_modified_time) {
+        res.setHeader('Last-Modified', time_utils.format_http_header_date(new Date(reply.last_modified_time)));
+    } else {
+        res.setHeader('Last-Modified', time_utils.format_http_header_date(new Date(reply.create_time)));
+    }
+    set_encryption_response_headers(req, res, reply.encryption);
+}
+
 /**
  * @param {nb.S3Response} res 
  * @param {Array<string>} [supported_storage_classes]
@@ -774,6 +799,7 @@ exports.parse_part_number = parse_part_number;
 exports.parse_copy_source = parse_copy_source;
 exports.format_copy_source = format_copy_source;
 exports.set_response_object_md = set_response_object_md;
+exports.set_response_headers_get_object_attributes = set_response_headers_get_object_attributes;
 exports.parse_storage_class = parse_storage_class;
 exports.parse_storage_class_header = parse_storage_class_header;
 exports.parse_encryption = parse_encryption;
@@ -801,3 +827,6 @@ exports.get_default_object_owner = get_default_object_owner;
 exports.set_response_supported_storage_classes = set_response_supported_storage_classes;
 exports.cont_tok_to_key_marker = cont_tok_to_key_marker;
 exports.key_marker_to_cont_tok = key_marker_to_cont_tok;
+exports.parse_sse_c = parse_sse_c;
+exports.OBJECT_ATTRIBUTES = OBJECT_ATTRIBUTES;
+exports.OBJECT_ATTRIBUTES_UNSUPPORTED = OBJECT_ATTRIBUTES_UNSUPPORTED;

src/sdk/namespace_fs.js

@@ -2390,7 +2390,8 @@ class NamespaceFS {
         const etag = this._get_etag(stat);
         const create_time = stat.mtime.getTime();
         const encryption = this._get_encryption_info(stat);
-        const version_id = (this._is_versioning_enabled() || this._is_versioning_suspended()) && this._get_version_id_by_xattr(stat);
+        const version_id = ((this._is_versioning_enabled() || this._is_versioning_suspended()) && this._get_version_id_by_xattr(stat)) ||
+            undefined;
         const delete_marker = stat.xattr?.[XATTR_DELETE_MARKER] === 'true';
         const dir_content_type = stat.xattr?.[XATTR_DIR_CONTENT] && ((Number(stat.xattr?.[XATTR_DIR_CONTENT]) > 0 && 'application/octet-stream') || 'application/x-directory');
         const content_type = stat.xattr?.[XATTR_CONTENT_TYPE] ||

src/sdk/namespace_s3.js

@@ -745,6 +745,43 @@ class NamespaceS3 {
         throw new S3Error(S3Error.NotImplemented);
     }
 
+    //////////////////////////
+    //  OBJECT ATTRIBUTES   //
+    //////////////////////////
+
+    async get_object_attributes(params, object_sdk) {
+        dbg.log0('NamespaceS3.get_object_attributes:', this.bucket, inspect(params));
+        await this._prepare_sts_client();
+
+        /** @type {AWS.S3.GetObjectAttributesRequest} */
+        const request = {
+            Bucket: this.bucket,
+            Key: params.key,
+            VersionId: params.version_id,
+            ObjectAttributes: params.attributes,
+        };
+        this._set_md_conditions(params, request);
+        this._assign_encryption_to_request(params, request);
+        try {
+            const res = await this.s3.getObjectAttributes(request).promise();
+            dbg.log0('NamespaceS3.get_object_attributes:', this.bucket, inspect(params), 'res', inspect(res));
+            return this._get_s3_object_info(res, params.bucket);
+        } catch (err) {
+            this._translate_error_code(params, err);
+            dbg.warn('NamespaceS3.get_object_attributes:', inspect(err));
+            // It's totally expected to issue `HeadObject` against an object that doesn't exist
+            // this shouldn't be counted as an issue for the namespace store
+            if (err.rpc_code !== 'NO_SUCH_OBJECT') {
+                object_sdk.rpc_client.pool.update_issues_report({
+                    namespace_resource_id: this.namespace_resource_id,
+                    error_code: String(err.code),
+                    time: Date.now(),
+                });
+            }
+            throw err;
+        }
+    }
+
     ///////////////
     // INTERNALS //
     ///////////////
@@ -756,7 +793,8 @@ class NamespaceS3 {
      *   AWS.S3.ObjectVersion &
      *   AWS.S3.DeleteMarkerEntry &
      *   AWS.S3.MultipartUpload &
-     *   AWS.S3.GetObjectOutput
+     *   AWS.S3.GetObjectOutput &
+     *   AWS.S3.GetObjectAttributesOutput
      * >, 'ChecksumAlgorithm'>} res 
      * @param {string} bucket 
      * @param {number} [part_number]
@@ -796,7 +834,10 @@ class NamespaceS3 {
             sha256_b64: undefined,
             stats: undefined,
             tagging: undefined,
-            object_owner: this._get_object_owner()
+            object_owner: this._get_object_owner(),
+            checksum: res.Checksum,
+            // @ts-ignore // See note in GetObjectAttributesParts in file nb.d.ts
+            object_parts: res.ObjectParts,
         };
     }
 

src/sdk/nb.d.ts

@@ -5,6 +5,7 @@ import * as mongodb from 'mongodb';
 import { EventEmitter } from 'events';
 import { Readable, Writable } from 'stream';
 import { IncomingMessage, ServerResponse } from 'http';
+import { ObjectPart, Checksum} from '@aws-sdk/client-s3';
 
 type Semaphore = import('../util/semaphore');
 type KeysSemaphore = import('../util/keys_semaphore');
@@ -439,6 +440,8 @@ interface ObjectInfo {
     ns?: Namespace;
     storage_class?: StorageClass;
     restore_status?: { ongoing?: boolean; expiry_time?: Date; };
+    checksum?: Checksum;
+    object_parts?: GetObjectAttributesParts;
 }
 
 
@@ -814,6 +817,7 @@ interface Namespace {
     get_blob_block_lists(params: object, object_sdk: ObjectSDK): Promise<any>;
 
     restore_object(params: object, object_sdk: ObjectSDK): Promise<any>;
+    get_object_attributes(params: object, object_sdk: ObjectSDK): Promise<any>;
 }
 
 interface BucketSpace {
@@ -1129,3 +1133,20 @@ interface RestoreStatus {
     ongoing?: boolean;
     expiry_time?: Date;
 }
+
+/**********************************************************
+ *
+ * OTHER - S3 Structure
+ *
+ **********************************************************/
+
+// Since the interface is a bit different between the SDKs
+// we couldn't import and reuse
+interface GetObjectAttributesParts {
+    TotalPartsCount?: number;
+    PartNumberMarker?: string; // in AWS SDK V2 it is number
+    NextPartNumberMarker?: string; // in AWS SDK V2 it is number
+    MaxParts?: number;
+    IsTruncated?: boolean;
+    Parts?: ObjectPart[];
+  }

src/sdk/object_sdk.js

@@ -1127,6 +1127,24 @@ class ObjectSDK {
         this._check_is_readonly_namespace(ns);
         return ns.put_object_acl(params, this);
     }
+
+    //////////////////////////
+    //  OBJECT ATTRIBUTES   //
+    //////////////////////////
+
+    async get_object_attributes(params) {
+        const ns = await this._get_bucket_namespace(params.bucket);
+        if (ns.get_object_attributes) {
+            return ns.get_object_attributes(params, this);
+          } else {
+            // fallback to calling get_object_md without attributes params
+            dbg.warn('namespace does not implement get_object_attributes action, fallback to read_object_md');
+            const md_params = { ...params };
+            delete md_params.attributes; // not part of the schema of read_object_md
+            return ns.read_object_md(md_params, this);
+          }
+    }
+
 }
 
 // EXPORT