feat userver: refactor ReadRecursiveFilesInfoWithData method

Tests: checked in gtests
commit_hash:2177960320dcdda692eb0146f30b532ff6502991

Author: basil-shuman

.mapping.json

@@ -1701,7 +1701,6 @@
   "core/src/formats/json/stack_overflow_test.cpp":"taxi/uservices/userver/core/src/formats/json/stack_overflow_test.cpp",
   "core/src/fs/fs_cache_client.cpp":"taxi/uservices/userver/core/src/fs/fs_cache_client.cpp",
   "core/src/fs/read.cpp":"taxi/uservices/userver/core/src/fs/read.cpp",
-  "core/src/fs/read_test.cpp":"taxi/uservices/userver/core/src/fs/read_test.cpp",
   "core/src/fs/temp_file.cpp":"taxi/uservices/userver/core/src/fs/temp_file.cpp",
   "core/src/fs/temp_file_test.cpp":"taxi/uservices/userver/core/src/fs/temp_file_test.cpp",
   "core/src/fs/write.cpp":"taxi/uservices/userver/core/src/fs/write.cpp",
@@ -5224,6 +5223,9 @@
   "universal/include/userver/fs/blocking/temp_directory.hpp":"taxi/uservices/userver/universal/include/userver/fs/blocking/temp_directory.hpp",
   "universal/include/userver/fs/blocking/temp_file.hpp":"taxi/uservices/userver/universal/include/userver/fs/blocking/temp_file.hpp",
   "universal/include/userver/fs/blocking/write.hpp":"taxi/uservices/userver/universal/include/userver/fs/blocking/write.hpp",
+  "universal/include/userver/fs/file_info_with_data.hpp":"taxi/uservices/userver/universal/include/userver/fs/file_info_with_data.hpp",
+  "universal/include/userver/fs/path_utils.hpp":"taxi/uservices/userver/universal/include/userver/fs/path_utils.hpp",
+  "universal/include/userver/fs/settings_read_file.hpp":"taxi/uservices/userver/universal/include/userver/fs/settings_read_file.hpp",
   "universal/include/userver/hostinfo/blocking/get_hostname.hpp":"taxi/uservices/userver/universal/include/userver/hostinfo/blocking/get_hostname.hpp",
   "universal/include/userver/hostinfo/blocking/read_groups.hpp":"taxi/uservices/userver/universal/include/userver/hostinfo/blocking/read_groups.hpp",
   "universal/include/userver/hostinfo/cpu_limit.hpp":"taxi/uservices/userver/universal/include/userver/hostinfo/cpu_limit.hpp",
@@ -5490,12 +5492,15 @@
   "universal/src/fs/blocking/file_descriptor.cpp":"taxi/uservices/userver/universal/src/fs/blocking/file_descriptor.cpp",
   "universal/src/fs/blocking/file_descriptor_test.cpp":"taxi/uservices/userver/universal/src/fs/blocking/file_descriptor_test.cpp",
   "universal/src/fs/blocking/read.cpp":"taxi/uservices/userver/universal/src/fs/blocking/read.cpp",
+  "universal/src/fs/blocking/read_test.cpp":"taxi/uservices/userver/universal/src/fs/blocking/read_test.cpp",
   "universal/src/fs/blocking/temp_directory.cpp":"taxi/uservices/userver/universal/src/fs/blocking/temp_directory.cpp",
   "universal/src/fs/blocking/temp_directory_test.cpp":"taxi/uservices/userver/universal/src/fs/blocking/temp_directory_test.cpp",
   "universal/src/fs/blocking/temp_file.cpp":"taxi/uservices/userver/universal/src/fs/blocking/temp_file.cpp",
   "universal/src/fs/blocking/temp_file_test.cpp":"taxi/uservices/userver/universal/src/fs/blocking/temp_file_test.cpp",
   "universal/src/fs/blocking/write.cpp":"taxi/uservices/userver/universal/src/fs/blocking/write.cpp",
   "universal/src/fs/blocking/write_test.cpp":"taxi/uservices/userver/universal/src/fs/blocking/write_test.cpp",
+  "universal/src/fs/path_utils.cpp":"taxi/uservices/userver/universal/src/fs/path_utils.cpp",
+  "universal/src/fs/path_utils_test.cpp":"taxi/uservices/userver/universal/src/fs/path_utils_test.cpp",
   "universal/src/hostinfo/blocking/get_hostname.cpp":"taxi/uservices/userver/universal/src/hostinfo/blocking/get_hostname.cpp",
   "universal/src/hostinfo/blocking/get_hostname_test.cpp":"taxi/uservices/userver/universal/src/hostinfo/blocking/get_hostname_test.cpp",
   "universal/src/hostinfo/blocking/read_groups.cpp":"taxi/uservices/userver/universal/src/hostinfo/blocking/read_groups.cpp",

core/include/userver/fs/read.hpp

@@ -3,40 +3,18 @@
 /// @file userver/fs/read.hpp
 /// @brief functions for asynchronous file read operations
 
-#include <memory>
 #include <string>
-#include <unordered_map>
 
 #include <userver/engine/task/task_processor_fwd.hpp>
-#include <userver/utils/flags.hpp>
+#include <userver/fs/file_info_with_data.hpp>
+#include <userver/fs/path_utils.hpp>
+#include <userver/fs/settings_read_file.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
 /// @brief filesystem support
 namespace fs {
 
-/// @brief Struct file with load data
-struct FileInfoWithData {
-    std::string data;
-    std::string extension;
-};
-
-using FileInfoWithDataConstPtr = std::shared_ptr<const FileInfoWithData>;
-using FileInfoWithDataMap = std::unordered_map<std::string, FileInfoWithDataConstPtr>;
-
-enum class SettingsReadFile {
-    kNone = 0,
-    /// Skip hidden files,
-    kSkipHidden = 1 << 0,
-};
-
-/// @brief Returns relative path from full path
-/// @param path full path, must start with `dir`
-/// @param dir directory path to get relative path
-/// @note it does not access filesystem, the relative path is calculated
-/// lexically.
-std::string GetLexicallyRelative(std::string_view path, std::string_view dir);
-
 /// @brief Returns files from recursively traversed directory
 /// @param async_tp TaskProcessor for synchronous waiting
 /// @param path to directory to traverse recursively
@@ -47,7 +25,7 @@ std::string GetLexicallyRelative(std::string_view path, std::string_view dir);
 FileInfoWithDataMap ReadRecursiveFilesInfoWithData(
     engine::TaskProcessor& async_tp,
     const std::string& path,
-    utils::Flags<SettingsReadFile> flags = {SettingsReadFile::kSkipHidden}
+    SettingReadFileFlags flags = {SettingsReadFile::kSkipHidden}
 );
 
 /// @brief Reads file contents asynchronously

core/src/fs/read.cpp

@@ -1,62 +1,22 @@
 #include <userver/fs/read.hpp>
 
-#include <boost/filesystem.hpp>
-
 #include <userver/engine/async.hpp>
 #include <userver/fs/blocking/read.hpp>
-#include <userver/utils/async.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
 namespace fs {
 
-namespace {
-
-bool IsHiddenFile(const boost::filesystem::path& path) {
-    auto name = path.filename().native();
-    UASSERT(!name.empty());
-    return name != ".." && name != "." && name[0] == '.';
-}
-
-}  // namespace
-
-std::string GetLexicallyRelative(std::string_view path, std::string_view dir) {
-    UASSERT(dir.size() < path.size());
-    UASSERT(path.substr(0, dir.size()) == dir);
-    auto rel = path.substr(dir.size());
-    return std::string{rel};
-}
-
 std::string ReadFileContents(engine::TaskProcessor& async_tp, const std::string& path) {
     return engine::AsyncNoSpan(async_tp, &fs::blocking::ReadFileContents, path).Get();
 }
 
 FileInfoWithDataMap ReadRecursiveFilesInfoWithData(
     engine::TaskProcessor& async_tp,
     const std::string& path,
-    utils::Flags<SettingsReadFile> flags
+    SettingReadFileFlags flags
 ) {
-    FileInfoWithDataMap data{};
-    for (auto it =
-             utils::Async(async_tp, "init", [&path] { return boost::filesystem::recursive_directory_iterator(path); })
-                 .Get();
-         it != boost::filesystem::recursive_directory_iterator();
-         utils::Async(async_tp, "next", [&it] { ++it; }).Get())
-    {
-        // only files
-        if (it->status().type() != boost::filesystem::regular_file) {
-            continue;
-        }
-        if ((flags & SettingsReadFile::kSkipHidden) && IsHiddenFile(it->path())) {
-            continue;
-        }
-        FileInfoWithData info{};
-        info.extension = it->path().extension().string();
-        info.data = ReadFileContents(async_tp, it->path().string());
-        data[GetLexicallyRelative(it->path().string(), path)] = std::make_shared<const FileInfoWithData>(std::move(info)
-        );
-    }
-    return data;
+    return engine::AsyncNoSpan(async_tp, &fs::blocking::ReadRecursiveFilesInfoWithData, std::cref(path), flags).Get();
 }
 
 bool FileExists(engine::TaskProcessor& async_tp, const std::string& path) {

universal/include/userver/fs/blocking/read.hpp

@@ -6,6 +6,8 @@
 
 #include <string>
 
+#include <userver/fs/file_info_with_data.hpp>
+#include <userver/fs/settings_read_file.hpp>
 #include <userver/utils/boost_filesystem_file_status.hpp>
 
 USERVER_NAMESPACE_BEGIN
@@ -37,6 +39,14 @@ bool FileExists(const std::string& path);
 /// @throws std::runtime_error if something goes wrong
 boost::filesystem::file_type GetFileType(const std::string& path);
 
+/// @brief Returns files from recursively traversed directory
+/// @param path to directory to traverse recursively
+/// @param flags settings read files
+/// @returns map with relative to `path` filepaths and file info
+/// @throws std::runtime_error if read fails for any reason (e.g. no such file,
+/// read error, etc.),
+FileInfoWithDataMap ReadRecursiveFilesInfoWithData(const std::string& path, SettingReadFileFlags flags);
+
 }  // namespace fs::blocking
 
 USERVER_NAMESPACE_END

universal/include/userver/fs/file_info_with_data.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+/// @file userver/fs/file_info_with_data.hpp
+/// @brief data structures to store file info with load data
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+USERVER_NAMESPACE_BEGIN
+
+/// @brief filesystem support
+namespace fs {
+
+/// @brief Struct file with load data
+struct FileInfoWithData {
+    std::string data;
+    std::string extension;
+};
+
+using FileInfoWithDataConstPtr = std::shared_ptr<const FileInfoWithData>;
+using FileInfoWithDataMap = std::unordered_map<std::string, FileInfoWithDataConstPtr>;
+
+}  // namespace fs
+
+USERVER_NAMESPACE_END

universal/include/userver/fs/path_utils.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+/// @file userver/fs/path_utils.hpp
+/// @brief file path operations and utilities
+
+#include <string>
+
+USERVER_NAMESPACE_BEGIN
+
+/// @brief filesystem support
+namespace fs {
+
+/// @brief Returns relative path from full path
+/// @param path full path, must start with `dir`
+/// @param dir directory path to get relative path
+/// @note it does not access filesystem, the relative path is calculated
+/// lexically.
+std::string GetLexicallyRelative(std::string_view path, std::string_view dir);
+
+}  // namespace fs
+
+USERVER_NAMESPACE_END

universal/include/userver/fs/settings_read_file.hpp

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <userver/utils/flags.hpp>
+
+/// @file userver/fs/settings_read_file.hpp
+/// @brief settings flags for directory recursive read operations
+
+USERVER_NAMESPACE_BEGIN
+
+namespace fs {
+
+/// @brief filesystem support
+enum class SettingsReadFile {
+    kNone = 0,
+    /// Skip hidden files,
+    kSkipHidden = 1 << 0,
+};
+
+using SettingReadFileFlags = utils::Flags<SettingsReadFile>;
+
+}  // namespace fs
+
+USERVER_NAMESPACE_END

universal/src/fs/blocking/read.cpp

@@ -1,14 +1,27 @@
 #include <userver/fs/blocking/read.hpp>
+#include <userver/fs/path_utils.hpp>
+#include <userver/utils/assert.hpp>
 
 #include <fstream>
 #include <sstream>
 
+#include <boost/filesystem/directory.hpp>
 #include <boost/filesystem/operations.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
 namespace fs::blocking {
 
+namespace {
+
+bool IsHiddenFile(const boost::filesystem::path& path) {
+    auto name = path.filename().native();
+    UASSERT(!name.empty());
+    return name != ".." && name != "." && name[0] == '.';
+}
+
+}  // namespace
+
 std::string ReadFileContents(const std::string& path) {
     const std::ifstream ifs(path);
     if (!ifs) {
@@ -24,6 +37,28 @@ bool FileExists(const std::string& path) { return boost::filesystem::exists(path
 
 boost::filesystem::file_type GetFileType(const std::string& path) { return boost::filesystem::status(path).type(); }
 
+FileInfoWithDataMap ReadRecursiveFilesInfoWithData(const std::string& path, utils::Flags<SettingsReadFile> flags) {
+    FileInfoWithDataMap data{};
+    for (auto it = boost::filesystem::recursive_directory_iterator(path);
+         it != boost::filesystem::recursive_directory_iterator();
+         ++it)
+    {
+        // only files
+        if (it->status().type() != boost::filesystem::regular_file) {
+            continue;
+        }
+        if ((flags & SettingsReadFile::kSkipHidden) && IsHiddenFile(it->path())) {
+            continue;
+        }
+        FileInfoWithData info{};
+        info.extension = it->path().extension().string();
+        info.data = ReadFileContents(it->path().string());
+        data[GetLexicallyRelative(it->path().string(), path)] = std::make_shared<const FileInfoWithData>(std::move(info)
+        );
+    }
+    return data;
+}
+
 }  // namespace fs::blocking
 
 USERVER_NAMESPACE_END