diff --git a/src/sdk/glacier.js b/src/sdk/glacier.js index dcc589e287..d44647d00c 100644 --- a/src/sdk/glacier.js +++ b/src/sdk/glacier.js @@ -300,6 +300,32 @@ class Glacier { return restore_status.state === Glacier.RESTORE_STATUS_CAN_RESTORE; } + /** + * encode_log takes in data intended for the backend and encodes + * it. + * + * This method must be overwritten for all the backends if they need + * different encodings for their logs. + * @param {string} data + * @returns {string} + */ + encode_log(data) { + return data; + } + + /** + * decode_log takes in data intended for the backend and decodes + * it. + * + * This method must be overwritten for all the backends if they need + * different encodings for their logs. + * @param {string} data + * @returns {string} + */ + decode_log(data) { + return data; + } + /** * get_restore_status returns status of the object at the given * file_path diff --git a/src/sdk/glacier_tapecloud.js b/src/sdk/glacier_tapecloud.js index 9333eee1f4..a4e24cf6ec 100644 --- a/src/sdk/glacier_tapecloud.js +++ b/src/sdk/glacier_tapecloud.js @@ -195,6 +195,8 @@ class TapeCloudUtils { } class TapeCloudGlacier extends Glacier { + static LOG_DELIM = ' -- '; + /** * @param {nb.NativeFSContext} fs_context * @param {LogFile} log_file @@ -204,8 +206,14 @@ class TapeCloudGlacier extends Glacier { async stage_migrate(fs_context, log_file, failure_recorder) { dbg.log2('TapeCloudGlacier.stage_migrate starting for', log_file.log_path); + // Wrap failure recorder to make sure we correctly encode the entries + // before appending them to the failure log + const encoded_failure_recorder = async failure => failure_recorder(this.encode_log(failure)); + try { await log_file.collect(Glacier.MIGRATE_STAGE_WAL_NAME, async (entry, batch_recorder) => { + entry = this.decode_log(entry); + let entry_fh; let should_migrate = true; try { @@ -234,7 +242,7 @@ class TapeCloudGlacier extends Glacier { // Can't really do anything if this fails - provider // needs to make sure that appropriate error handling // is being done there - await failure_recorder(entry); + await encoded_failure_recorder(entry); return; } @@ -244,14 +252,14 @@ class TapeCloudGlacier extends Glacier { // Mark the file staged try { await entry_fh.replacexattr(fs_context, { [Glacier.XATTR_STAGE_MIGRATE]: Date.now().toString() }); - await batch_recorder(entry); + await batch_recorder(this.encode_log(entry)); } catch (error) { dbg.error('failed to mark the entry migrate staged', error); // Can't really do anything if this fails - provider // needs to make sure that appropriate error handling // is being done there - await failure_recorder(entry); + await encoded_failure_recorder(entry); } finally { entry_fh?.close(fs_context); } @@ -272,16 +280,23 @@ class TapeCloudGlacier extends Glacier { */ async migrate(fs_context, log_file, failure_recorder) { dbg.log2('TapeCloudGlacier.migrate starting for', log_file.log_path); + + // Wrap failure recorder to make sure we correctly encode the entries + // before appending them to the failure log + const encoded_failure_recorder = async failure => failure_recorder(this.encode_log(failure)); + try { // This will throw error only if our eeadm error handler // panics as well and at that point it's okay to // not handle the error and rather keep the log file around - await this._migrate(log_file.log_path, failure_recorder); + await this._migrate(log_file.log_path, encoded_failure_recorder); // Un-stage all the files - We don't need to deal with the cases // where some files have migrated and some have not as that is // not important for staging/un-staging. await log_file.collect_and_process(async entry => { + entry = this.decode_log(entry); + let fh; try { fh = await nb_native().fs.open(fs_context, entry); @@ -297,7 +312,7 @@ class TapeCloudGlacier extends Glacier { // Add the enty to the failure log - This could be wasteful as it might // add entries which have already been migrated but this is a better // retry. - await failure_recorder(entry); + await encoded_failure_recorder(entry); } finally { await fh?.close(fs_context); } @@ -319,8 +334,14 @@ class TapeCloudGlacier extends Glacier { async stage_restore(fs_context, log_file, failure_recorder) { dbg.log2('TapeCloudGlacier.stage_restore starting for', log_file.log_path); + // Wrap failure recorder to make sure we correctly encode the entries + // before appending them to the failure log + const encoded_failure_recorder = async failure => failure_recorder(this.encode_log(failure)); + try { await log_file.collect(Glacier.RESTORE_STAGE_WAL_NAME, async (entry, batch_recorder) => { + entry = this.decode_log(entry); + let fh; try { fh = await nb_native().fs.open(fs_context, entry); @@ -347,9 +368,9 @@ class TapeCloudGlacier extends Glacier { // 3. If we read corrupt value then either the file is getting staged or is // getting un-staged - In either case we must requeue. if (stat.xattr[Glacier.XATTR_STAGE_MIGRATE]) { - await failure_recorder(entry); + await encoded_failure_recorder(entry); } else { - await batch_recorder(entry); + await batch_recorder(this.encode_log(entry)); } } catch (error) { if (error.code === 'ENOENT') { @@ -361,7 +382,7 @@ class TapeCloudGlacier extends Glacier { 'adding log entry', entry, 'to failure recorder due to error', error, ); - await failure_recorder(entry); + await encoded_failure_recorder(entry); } finally { await fh?.close(fs_context); } @@ -383,16 +404,22 @@ class TapeCloudGlacier extends Glacier { async restore(fs_context, log_file, failure_recorder) { dbg.log2('TapeCloudGlacier.restore starting for', log_file.log_path); + // Wrap failure recorder to make sure we correctly encode the entries + // before appending them to the failure log + const encoded_failure_recorder = async failure => failure_recorder(this.encode_log(failure)); + try { const success = await this._recall( log_file.log_path, async entry_path => { + entry_path = this.decode_log(entry_path); dbg.log2('TapeCloudGlacier.restore.partial_failure - entry:', entry_path); - await failure_recorder(entry_path); + await encoded_failure_recorder(entry_path); }, async entry_path => { + entry_path = this.decode_log(entry_path); dbg.log2('TapeCloudGlacier.restore.partial_success - entry:', entry_path); - await this._finalize_restore(fs_context, entry_path, failure_recorder); + await this._finalize_restore(fs_context, entry_path, encoded_failure_recorder); } ); @@ -400,8 +427,9 @@ class TapeCloudGlacier extends Glacier { // the recall call. if (success) { await log_file.collect_and_process(async (entry_path, batch_recorder) => { + entry_path = this.decode_log(entry_path); dbg.log2('TapeCloudGlacier.restore.batch - entry:', entry_path); - await this._finalize_restore(fs_context, entry_path, failure_recorder); + await this._finalize_restore(fs_context, entry_path, encoded_failure_recorder); }); } @@ -425,6 +453,25 @@ class TapeCloudGlacier extends Glacier { return result.toLowerCase().trim() === 'true'; } + /** + * + * @param {string} data + * @returns + */ + encode_log(data) { + const encoded = data.replace(/\\/g, '\\\\').replace(/\n/g, '\\n'); + return `${TapeCloudGlacier.LOG_DELIM}${encoded}`; + } + + /** + * + * @param {string} data + * @returns + */ + decode_log(data) { + return data.substring(TapeCloudGlacier.LOG_DELIM.length).replace(/\\n/g, '\n').replace(/\\\\/g, '\\'); + } + // ============= PRIVATE FUNCTIONS ============= /** diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js index 986f9fcc8d..08c2f34f58 100644 --- a/src/sdk/namespace_fs.js +++ b/src/sdk/namespace_fs.js @@ -3697,13 +3697,13 @@ class NamespaceFS { async append_to_migrate_wal(entry) { if (!config.NSFS_GLACIER_LOGS_ENABLED) return; - await NamespaceFS.migrate_wal.append(entry); + await NamespaceFS.migrate_wal.append(Glacier.getBackend().encode_log(entry)); } async append_to_restore_wal(entry) { if (!config.NSFS_GLACIER_LOGS_ENABLED) return; - await NamespaceFS.restore_wal.append(entry); + await NamespaceFS.restore_wal.append(Glacier.getBackend().encode_log(entry)); } static get migrate_wal() { diff --git a/src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js b/src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js index 63a742e756..7be970aeda 100644 --- a/src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js +++ b/src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js @@ -166,6 +166,8 @@ mocha.describe('nsfs_glacier', function() { }); mocha.describe('nsfs_glacier_tapecloud', async function() { + const restore_key_spcl_char_1 = 'restore_key_2_\n'; + const restore_key_spcl_char_2 = 'restore_key_2_\n_2'; const upload_key = 'upload_key_1'; const restore_key = 'restore_key_1'; const xattr = { key: 'value', key2: 'value2' }; @@ -280,10 +282,10 @@ mocha.describe('nsfs_glacier', function() { failure_backend._process_expired = async () => { /**noop*/ }; failure_backend._recall = async (_file, failure_recorder, success_recorder) => { // This unintentionally also replicates duplicate entries in WAL - await failure_recorder(failed_file_path); + await failure_recorder(failure_backend.encode_log(failed_file_path)); // This unintentionally also replicates duplicate entries in WAL - await success_recorder(success_file_path); + await success_recorder(failure_backend.encode_log(success_file_path)); return false; }; @@ -324,6 +326,126 @@ mocha.describe('nsfs_glacier', function() { assert(failure_stats.xattr[Glacier.XATTR_RESTORE_REQUEST]); }); + mocha.it('restore-object should successfully restore objects with special characters', async function() { + const now = Date.now(); + const data = crypto.randomBytes(100); + const all_params = [ + { + bucket: upload_bkt, + key: restore_key_spcl_char_1, + storage_class: s3_utils.STORAGE_CLASS_GLACIER, + xattr, + days: 1, + source_stream: buffer_utils.buffer_to_read_stream(data) + }, + { + bucket: upload_bkt, + key: restore_key_spcl_char_2, + storage_class: s3_utils.STORAGE_CLASS_GLACIER, + xattr, + days: 1, + source_stream: buffer_utils.buffer_to_read_stream(data) + } + ]; + + for (const params of all_params) { + const upload_res = await glacier_ns.upload_object(params, dummy_object_sdk); + console.log('upload_object response', inspect(upload_res)); + + const restore_res = await glacier_ns.restore_object(params, dummy_object_sdk); + assert(restore_res); + + // Issue restore + await backend.perform(glacier_ns.prepare_fs_context(dummy_object_sdk), "RESTORE"); + + + // Ensure object is restored + const md = await glacier_ns.read_object_md(params, dummy_object_sdk); + + assert(!md.restore_status.ongoing); + + const expected_expiry = Glacier.generate_expiry(new Date(), params.days, '', config.NSFS_GLACIER_EXPIRY_TZ); + assert(expected_expiry.getTime() >= md.restore_status.expiry_time.getTime()); + assert(now <= md.restore_status.expiry_time.getTime()); + } + }); + + mocha.it('restore-object should not restore failed item with special characters', async function() { + const now = Date.now(); + const data = crypto.randomBytes(100); + const failed_restore_key = `${restore_key_spcl_char_1}_failured`; + const success_restore_key = `${restore_key_spcl_char_1}_success`; + + const failed_params = { + bucket: upload_bkt, + key: failed_restore_key, + storage_class: s3_utils.STORAGE_CLASS_GLACIER, + xattr, + days: 1, + source_stream: buffer_utils.buffer_to_read_stream(data) + }; + + const success_params = { + bucket: upload_bkt, + key: success_restore_key, + storage_class: s3_utils.STORAGE_CLASS_GLACIER, + xattr, + days: 1, + source_stream: buffer_utils.buffer_to_read_stream(data) + }; + + const failed_file_path = glacier_ns._get_file_path(failed_params); + const success_file_path = glacier_ns._get_file_path(success_params); + + const failure_backend = new TapeCloudGlacier(); + failure_backend._migrate = async () => true; + failure_backend._process_expired = async () => { /**noop*/ }; + failure_backend._recall = async (_file, failure_recorder, success_recorder) => { + // This unintentionally also replicates duplicate entries in WAL + await failure_recorder(failure_backend.encode_log(failed_file_path)); + + // This unintentionally also replicates duplicate entries in WAL + await success_recorder(failure_backend.encode_log(success_file_path)); + + return false; + }; + + const upload_res_1 = await glacier_ns.upload_object(failed_params, dummy_object_sdk); + console.log('upload_object response', inspect(upload_res_1)); + + const upload_res_2 = await glacier_ns.upload_object(success_params, dummy_object_sdk); + console.log('upload_object response', inspect(upload_res_2)); + + const restore_res_1 = await glacier_ns.restore_object(failed_params, dummy_object_sdk); + assert(restore_res_1); + + const restore_res_2 = await glacier_ns.restore_object(success_params, dummy_object_sdk); + assert(restore_res_2); + + const fs_context = glacier_ns.prepare_fs_context(dummy_object_sdk); + + // Issue restore + await failure_backend.perform(glacier_ns.prepare_fs_context(dummy_object_sdk), "RESTORE"); + + // Ensure success object is restored + const success_md = await glacier_ns.read_object_md(success_params, dummy_object_sdk); + + assert(!success_md.restore_status.ongoing); + + const expected_expiry = Glacier.generate_expiry(new Date(), success_params.days, '', config.NSFS_GLACIER_EXPIRY_TZ); + assert(expected_expiry.getTime() >= success_md.restore_status.expiry_time.getTime()); + assert(now <= success_md.restore_status.expiry_time.getTime()); + + // Ensure failed object is NOT restored + const failure_stats = await nb_native().fs.stat( + fs_context, + failed_file_path, + ); + + assert(!failure_stats.xattr[Glacier.XATTR_RESTORE_EXPIRY] || failure_stats.xattr[Glacier.XATTR_RESTORE_EXPIRY] === ''); + assert(failure_stats.xattr[Glacier.XATTR_RESTORE_REQUEST]); + }); + mocha.it('_finalize_restore should tolerate deleted objects', async function() { // should not throw error if the path does not exist await backend._finalize_restore(glacier_ns.prepare_fs_context(dummy_object_sdk), '/path/does/not/exist');