Merge branch 'feature/CLDSRV-744-check-content-md5-2' into q/9.0

Author: bert-e

config.json

@@ -142,5 +142,8 @@
     },
     "kmip": {
         "providerName": "thales"
+    },
+    "integrityChecks": {
+        "objectPutRetention": true
     }
 }

lib/Config.js

@@ -533,6 +533,37 @@ function bucketNotifAssert(bucketNotifConfig) {
     return bucketNotifConfig;
 }
 
+function parseIntegrityChecks(config) {
+    const integrityChecks = {
+        'bucketPutACL': true,
+        'bucketPutCors': true,
+        'bucketPutEncryption': true,
+        'bucketPutLifecycle': true,
+        'bucketPutNotification': true,
+        'bucketPutObjectLock': true,
+        'bucketPutPolicy': true,
+        'bucketPutReplication': true,
+        'bucketPutVersioning': true,
+        'bucketPutWebsite': true,
+        'multiObjectDelete': true,
+        'objectPutACL': true,
+        'objectPutLegalHold': true,
+        'objectPutTagging': true,
+        'objectPutRetention': true,
+    };
+
+    if (config && config.integrityChecks) {
+        for (const method in integrityChecks) {
+            if (method in config.integrityChecks) {
+                assert(typeof config.integrityChecks[method] == 'boolean', `bad config: ${method} not boolean`);
+                integrityChecks[method] = config.integrityChecks[method];
+            }
+        }
+    }
+
+    return integrityChecks;
+}
+
 /**
  * Reads from a config file and returns the content as a config object
  */
@@ -1743,6 +1774,8 @@ class Config extends EventEmitter {
 
         this.supportedLifecycleRules = parseSupportedLifecycleRules(config.supportedLifecycleRules);
 
+        this.integrityChecks = parseIntegrityChecks(config);
+
         /**
          * S3C-10336: PutObject max size of 5GB is new in 9.5.1
          * Provides a way to bypass the new validation if it breaks customer workflows
@@ -2081,4 +2114,5 @@ module.exports = {
     azureGetStorageAccountName,
     azureGetLocationCredentials,
     parseSupportedLifecycleRules,
+    parseIntegrityChecks,
 };

lib/api/api.js

@@ -74,6 +74,7 @@ const parseCopySource = require('./apiUtils/object/parseCopySource');
 const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
 const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
 const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
+const { validateMethodChecksumNoChunking } = require('./apiUtils/integrity/validateChecksums');
 
 const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
 
@@ -242,8 +243,16 @@ const api = {
                                   { postLength });
                         return next(errors.InvalidRequest);
                     }
+
+                    const buff = Buffer.concat(post, postLength);
+
+                    const err = validateMethodChecksumNoChunking(request, buff, log);
+                    if (err) {
+                        return next(err);
+                    }
+
                     // Convert array of post buffers into one string
-                    request.post = Buffer.concat(post, postLength).toString();
+                    request.post = buff.toString();
                     return next(null, userInfo, authorizationResults, streamingV4Params, infos);
                 });
                 return undefined;

lib/api/apiUtils/integrity/validateChecksums.js

@@ -0,0 +1,82 @@
+const crypto = require('crypto');
+const { errors: ArsenalErrors } = require('arsenal');
+const { config } = require('../../../Config');
+
+const ChecksumError = Object.freeze({
+    MD5Mismatch: 'MD5Mismatch',
+    MissingChecksum: 'MissingChecksum',
+});
+
+/**
+ * validateChecksumsNoChunking - Validate the checksums of a request.
+ * @param {object} headers - http headers
+ * @param {Buffer} body - http request body
+ * @return {object} - error
+ */
+function validateChecksumsNoChunking(headers, body) {
+    if (headers && 'content-md5' in headers) {
+        const md5 = crypto.createHash('md5').update(body).digest('base64');
+        if (md5 !== headers['content-md5']) {
+            return { error: ChecksumError.MD5Mismatch, details: { calculated: md5, expected: headers['content-md5'] } };
+        }
+
+        return null;
+    }
+
+    return { error: ChecksumError.MissingChecksum, details: null };
+}
+
+function defaultValidationFunc(request, body, log) {
+    const err = validateChecksumsNoChunking(request.headers, body);
+    if (err && err.error !== ChecksumError.MissingChecksum) {
+        log.debug('failed checksum validation', { method: request.apiMethod }, err);
+        return ArsenalErrors.BadDigest;
+    }
+
+    return null;
+}
+
+const methodValidationFunc = Object.freeze({
+    'bucketPutACL': defaultValidationFunc,
+    'bucketPutCors': defaultValidationFunc,
+    'bucketPutEncryption': defaultValidationFunc,
+    'bucketPutLifecycle': defaultValidationFunc,
+    'bucketPutNotification': defaultValidationFunc,
+    'bucketPutObjectLock': defaultValidationFunc,
+    'bucketPutPolicy': defaultValidationFunc,
+    'bucketPutReplication': defaultValidationFunc,
+    'bucketPutVersioning': defaultValidationFunc,
+    'bucketPutWebsite': defaultValidationFunc,
+    // TODO: DeleteObjects requires a checksum. Should return an error if ChecksumError.MissingChecksum.
+    'multiObjectDelete': defaultValidationFunc,
+    'objectPutACL': defaultValidationFunc,
+    'objectPutLegalHold': defaultValidationFunc,
+    'objectPutTagging': defaultValidationFunc,
+    'objectPutRetention': defaultValidationFunc,
+});
+
+/**
+ * validateMethodChecksumsNoChunking - Validate the checksums of a request.
+ * @param {object} request - http request
+ * @param {Buffer} body - http request body
+ * @param {object} log - logger
+ * @return {object} - error
+ */
+function validateMethodChecksumNoChunking(request, body, log) {
+    if (config.integrityChecks[request.apiMethod]) {
+        const validationFunc = methodValidationFunc[request.apiMethod];
+        if (!validationFunc) {
+            return null;
+        }
+
+        return validationFunc(request, body, log);
+    }
+
+    return null;
+}
+
+module.exports = {
+    ChecksumError,
+    validateChecksumsNoChunking,
+    validateMethodChecksumNoChunking,
+};

lib/api/bucketPutCors.js

@@ -1,4 +1,3 @@
-const crypto = require('crypto');
 const async = require('async');
 const { errors, errorInstances } = require('arsenal');
 
@@ -33,16 +32,6 @@ function bucketPutCors(authInfo, request, log, callback) {
         return callback(errors.MissingRequestBodyError);
     }
 
-    if (request.headers['content-md5']) {
-        const md5 = crypto.createHash('md5')
-            .update(request.post, 'utf8').digest('base64');
-        if (md5 !== request.headers['content-md5']) {
-            log.debug('bad md5 digest', { error: errors.BadDigest });
-            monitoring.promMetrics('PUT', bucketName, 400, 'putBucketCors');
-            return callback(errors.BadDigest);
-        }
-    }
-
     if (parseInt(request.headers['content-length'], 10) > 65536) {
         const errMsg = 'The CORS XML document is limited to 64 KB in size.';
         log.debug(errMsg, { error: errors.MalformedXML });

lib/api/bucketPutReplication.js

@@ -33,6 +33,7 @@ function bucketPutReplication(authInfo, request, log, callback) {
         requestType: request.apiMethods || 'bucketPutReplication',
         request,
     };
+
     return waterfall([
         // Validate the request XML and return the replication configuration.
         next => getReplicationConfiguration(post, log, next),

lib/api/multiObjectDelete.js

@@ -1,5 +1,3 @@
-const crypto = require('crypto');
-
 const async = require('async');
 const { parseString } = require('xml2js');
 const { auth, errors, versioning, s3middleware, policies } = require('arsenal');
@@ -475,16 +473,6 @@ function multiObjectDelete(authInfo, request, log, callback) {
         return callback(errors.MissingRequestBodyError);
     }
 
-    if (request.headers['content-md5']) {
-        const md5 = crypto.createHash('md5')
-            .update(request.post, 'utf8').digest('base64');
-        if (md5 !== request.headers['content-md5']) {
-            monitoring.promMetrics('DELETE', request.bucketName, 400,
-                'multiObjectDelete');
-            return callback(errors.BadDigest);
-        }
-    }
-
     const inPlayInternal = [];
     const bucketName = request.bucketName;
     const canonicalID = authInfo.getCanonicalID();

lib/api/objectPutTagging.js

@@ -27,7 +27,6 @@ function objectPutTagging(authInfo, request, log, callback) {
     log.debug('processing request', { method: 'objectPutTagging' });
 
     const { bucketName, objectKey } = request;
-
     const decodedVidResult = decodeVersionId(request.query);
     if (decodedVidResult instanceof Error) {
         log.trace('invalid versionId query', {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zenko/cloudserver",
-  "version": "9.0.26",
+  "version": "9.0.27",
   "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol",
   "main": "index.js",
   "engines": {

tests/unit/Config.js

@@ -7,6 +7,7 @@ const {
     azureGetLocationCredentials,
     locationConstraintAssert,
     parseSupportedLifecycleRules,
+    parseIntegrityChecks,
     ConfigObject,
 } = require('../../lib/Config');
 
@@ -885,4 +886,40 @@ describe('Config', () => {
             assert.strictEqual(config.instanceId.length, 6);
         });
     });
+
+    describe('parse integrity checks', () => {
+        it('should replace default values with new values', () => {
+            const newConfig = {
+                integrityChecks: {
+                    'bucketPutACL': false,
+                    'bucketPutCors': false,
+                    'bucketPutEncryption': false,
+                    'bucketPutLifecycle': false,
+                    'bucketPutNotification': false,
+                    'bucketPutObjectLock': false,
+                    'bucketPutPolicy': false,
+                    'bucketPutReplication': false,
+                    'bucketPutVersioning': false,
+                    'bucketPutWebsite': false,
+                    'multiObjectDelete': false,
+                    'objectPutACL': false,
+                    'objectPutLegalHold': false,
+                    'objectPutTagging': false,
+                    'objectPutRetention': false,
+                },
+            };
+
+            const result = parseIntegrityChecks(newConfig);
+            for (const method in result) {
+                assert(result[method] == false, method);
+            }
+        });
+
+        it('default method value is true', () => {
+            const result = parseIntegrityChecks(null);
+            for (const method in result) {
+                assert(result[method] == true, method);
+            }
+        });
+    });
 });