rgw/d4n: write to the cache with O_SYNC flag by default

add conf to allow modifying cache write 'man 2 open' flags
with O_SYNC enabled by default.

performance impact measurment -
workload is PUT of 100K 5KB objects

- with O_SYNC flag: 3890 OP/s,
  iostat IO util% range while workload is running: ~45%-49%

- without O_SYNC flag: 4511 OP/s,
  iostat IO util% range while workload is running: ~10%-25%

Signed-off-by: Mark Kogan <[email protected]>

Author: mkogan1

src/common/options/rgw.yaml.in

@@ -3819,6 +3819,18 @@ options:
   services:
   - rgw
   with_legacy: true
+- name: rgw_d4n_l1_write_open_flags
+  type: uint
+  level: advanced
+  desc: cache files write 'man 2 open' 'file status flags' modifiers
+  long_desc: For example, to configure synchronized I/O, fcntl-linux.h defines (converted from octal)
+    O_SYNC = 1052672
+    O_DSYNC = 4096
+    (0 for no modification of the flags)
+  default: 1052672
+  services:
+  - rgw
+  with_legacy: true
 - name: rgw_d4n_libaio_aio_threads
   type: int
   level: advanced

src/rgw/rgw_ssd_driver.cc

@@ -671,7 +671,7 @@ int SSDDriver::AsyncWriteRequest::prepare_libaio_write_op(const DoutPrefixProvid
     cb.reset(new struct aiocb);
     memset(cb.get(), 0, sizeof(struct aiocb));
     mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
-    r = fd = TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode));
+    r = fd = TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | dpp->get_cct()->_conf->rgw_d4n_l1_write_open_flags, mode));
     if (fd < 0) {
         //directories might have been deleted by a parallel delete of the last version of an object
         if (errno == ENOENT) {
@@ -683,7 +683,7 @@ int SSDDriver::AsyncWriteRequest::prepare_libaio_write_op(const DoutPrefixProvid
             }
             ldpp_dout(dpp, 20) << "INFO: AsyncWriteRequest::prepare_libaio_write_op: dir_path for creating directories=" << dir_path << dendl;
             create_directories(dpp, dir_path);
-            r = fd = TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, mode));
+            r = fd = TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_WRONLY | O_CREAT | O_TRUNC | dpp->get_cct()->_conf->rgw_d4n_l1_write_open_flags, mode));
             if (fd < 0) {
                 ldpp_dout(dpp, 0) << "ERROR: AsyncWriteRequest::prepare_libaio_write_op: open file failed, errno=" << errno << ", location='" << file_path.c_str() << "'" << dendl;
                 return r;