Merge bitcoin/bitcoin#28076: util: Replace std::filesystem with util/fs.h

fanquake · fanquake · commit 63423480723d · 2023-11-13T14:10:54.000Z
bbbbdb0 ci: Add filesystem lint check (MarcoFalke) fada2f9 refactor: Replace <filesystem> with <util/fs.h> (MarcoFalke) Pull request description: Using `std::filesystem` is problematic: * There is a `fs` namespace wrapper for it. So having two ways to achieve the same is confusing. * Not using the `fs` wrapper is dangerous and buggy, because it disables known bugs by deleting problematic functions. Fix all issues by removing use of it and adding a linter to avoid using it again in the future. ACKs for top commit: TheCharlatan: ACK bbbbdb0 fanquake: ACK bbbbdb0 🦀 Tree-SHA512: 0e2d49742b08eb2635e6fce41485277cb9c40fe20b81017c391d3472a43787db1278a236825714ca1e41c9d2f59913865cfb0c649e3c8ab1fb598c849f80c660
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -84,6 +84,9 @@ task:
     memory: 1G
   # For faster CI feedback, immediately schedule the linters
   << : *CREDITS_TEMPLATE
+  test_runner_cache:
+    folder: "/lint_test_runner"
+    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:test/lint/test_runner)
   python_cache:
     folder: "/python_build"
     fingerprint_script: cat .python-version /etc/os-release
diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh
@@ -33,6 +33,17 @@ export PATH="${PYTHON_PATH}/bin:${PATH}"
 command -v python3
 python3 --version
 
+export LINT_RUNNER_PATH="/lint_test_runner"
+if [ ! -d "${LINT_RUNNER_PATH}" ]; then
+  ${CI_RETRY_EXE} apt-get install -y cargo
+  (
+    cd ./test/lint/test_runner || exit 1
+    cargo build
+    mkdir -p "${LINT_RUNNER_PATH}"
+    mv target/debug/test_runner "${LINT_RUNNER_PATH}"
+  )
+fi
+
 ${CI_RETRY_EXE} pip3 install \
   codespell==2.2.5 \
   flake8==6.1.0 \
diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh
@@ -30,6 +30,7 @@ test/lint/git-subtree-check.sh src/secp256k1
 test/lint/git-subtree-check.sh src/minisketch
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/git-subtree-check.sh src/crc32c
+RUST_BACKTRACE=1 "${LINT_RUNNER_PATH}/test_runner"
 test/lint/check-doc.py
 test/lint/all-lint.py
 
diff --git a/ci/lint/container-entrypoint.sh b/ci/lint/container-entrypoint.sh
@@ -11,6 +11,7 @@ export LC_ALL=C
 git config --global --add safe.directory /bitcoin
 
 export PATH="/python_build/bin:${PATH}"
+export LINT_RUNNER_PATH="/lint_test_runner"
 
 if [ -z "$1" ]; then
   LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh"
diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp
@@ -26,13 +26,13 @@
 #include <scheduler.h>
 #include <script/sigcache.h>
 #include <util/chaintype.h>
+#include <util/fs.h>
 #include <util/thread.h>
 #include <validation.h>
 #include <validationinterface.h>
 
 #include <cassert>
 #include <cstdint>
-#include <filesystem>
 #include <functional>
 #include <iosfwd>
 #include <memory>
@@ -50,8 +50,8 @@ int main(int argc, char* argv[])
             << "           BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl;
         return 1;
     }
-    std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]);
-    std::filesystem::create_directories(abs_datadir);
+    fs::path abs_datadir{fs::absolute(argv[1])};
+    fs::create_directories(abs_datadir);
 
 
     // SETUP: Context
diff --git a/src/common/args.cpp b/src/common/args.cpp
@@ -28,7 +28,6 @@
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
-#include <filesystem>
 #include <map>
 #include <optional>
 #include <stdexcept>
diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp
@@ -1044,8 +1044,8 @@ void CBlockPolicyEstimator::FlushUnconfirmed()
 
 std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge()
 {
-    auto file_time = std::filesystem::last_write_time(m_estimation_filepath);
-    auto now = std::filesystem::file_time_type::clock::now();
+    auto file_time{fs::last_write_time(m_estimation_filepath)};
+    auto now{fs::file_time_type::clock::now()};
     return std::chrono::duration_cast<std::chrono::hours>(now - file_time);
 }
 
diff --git a/src/util/fs.h b/src/util/fs.h
@@ -184,6 +184,7 @@ static inline path PathFromString(const std::string& string)
  * already exists or is a symlink to an existing directory.
  * This is a temporary workaround for an issue in libstdc++ that has been fixed
  * upstream [PR101510].
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510
  */
 static inline bool create_directories(const std::filesystem::path& p)
 {
diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp
@@ -11,13 +11,11 @@
 
 #include <logging.h>
 #include <sync.h>
-#include <tinyformat.h>
 #include <util/fs.h>
 #include <util/getuniquepath.h>
 #include <util/syserror.h>
 
 #include <cerrno>
-#include <filesystem>
 #include <fstream>
 #include <map>
 #include <memory>
@@ -263,7 +261,7 @@ bool RenameOver(fs::path src, fs::path dest)
 {
 #ifdef __MINGW64__
     // This is a workaround for a bug in libstdc++ which
-    // implements std::filesystem::rename with _wrename function.
+    // implements fs::rename with _wrename function.
     // This bug has been fixed in upstream:
     //  - GCC 10.3: 8dd1c1085587c9f8a21bb5e588dfe1e8cdbba79e
     //  - GCC 11.1: 1dfd95f0a0ca1d9e6cbc00e6cbfd1fa20a98f312
diff --git a/test/lint/README.md b/test/lint/README.md
@@ -13,6 +13,14 @@ DOCKER_BUILDKIT=1 docker build -t bitcoin-linter --file "./ci/lint_imagefile" ./
 Building the container can be done every time, because it is fast when the
 result is cached and it prevents issues when the image changes.
 
+test runner
+===========
+
+To run the checks in the test runner outside the docker, use:
+
+```sh
+( cd ./test/lint/test_runner/ && cargo fmt && cargo clippy && cargo run )
+```
 
 check-doc.py
 ============
diff --git a/test/lint/test_runner/Cargo.lock b/test/lint/test_runner/Cargo.lock
diff --git a/test/lint/test_runner/Cargo.toml b/test/lint/test_runner/Cargo.toml
@@ -0,0 +1,12 @@
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+
+[package]
+name = "test_runner"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs
@@ -0,0 +1,77 @@
+// Copyright (c) The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://opensource.org/license/mit/.
+
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+use std::process::ExitCode;
+
+use String as LintError;
+
+/// Return the git command
+fn git() -> Command {
+    Command::new("git")
+}
+
+/// Return stdout
+fn check_output(cmd: &mut std::process::Command) -> Result<String, LintError> {
+    let out = cmd.output().expect("command error");
+    if !out.status.success() {
+        return Err(String::from_utf8_lossy(&out.stderr).to_string());
+    }
+    Ok(String::from_utf8(out.stdout)
+        .map_err(|e| format!("{e}"))?
+        .trim()
+        .to_string())
+}
+
+/// Return the git root as utf8, or panic
+fn get_git_root() -> String {
+    check_output(git().args(["rev-parse", "--show-toplevel"])).unwrap()
+}
+
+fn lint_std_filesystem() -> Result<(), LintError> {
+    let found = git()
+        .args([
+            "grep",
+            "std::filesystem",
+            "--",
+            "./src/",
+            ":(exclude)src/util/fs.h",
+        ])
+        .status()
+        .expect("command error")
+        .success();
+    if found {
+        Err(r#"
+^^^
+Direct use of std::filesystem may be dangerous and buggy. Please include <util/fs.h> and use the
+fs:: namespace, which has unsafe filesystem functions marked as deleted.
+            "#
+        .to_string())
+    } else {
+        Ok(())
+    }
+}
+
+fn main() -> ExitCode {
+    let test_list = [("std::filesystem check", lint_std_filesystem)];
+
+    let git_root = PathBuf::from(get_git_root());
+
+    let mut test_failed = false;
+    for (lint_name, lint_fn) in test_list {
+        // chdir to root before each lint test
+        env::set_current_dir(&git_root).unwrap();
+        if let Err(err) = lint_fn() {
+            println!("{err}\n^---- Failure generated from {lint_name}!");
+            test_failed = true;
+        }
+    }
+    if test_failed {
+        ExitCode::FAILURE
+    } else {
+        ExitCode::SUCCESS
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -1044,8 +1044,8 @@ void CBlockPolicyEstimator::FlushUnconfirmed()`
`1044`	`1044`
`1045`	`1045`	`std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge()`
`1046`	`1046`	`{`
`1047`		`- auto file_time = std::filesystem::last_write_time(m_estimation_filepath);`
`1048`		`- auto now = std::filesystem::file_time_type::clock::now();`
	`1047`	`+ auto file_time{fs::last_write_time(m_estimation_filepath)};`
	`1048`	`+ auto now{fs::file_time_type::clock::now()};`
`1049`	`1049`	`return std::chrono::duration_cast<std::chrono::hours>(now - file_time);`
`1050`	`1050`	`}`
`1051`	`1051`
Original file line number	Diff line number	Diff line change
`@@ -184,6 +184,7 @@ static inline path PathFromString(const std::string& string)`
`184`	`184`	`* already exists or is a symlink to an existing directory.`
`185`	`185`	`* This is a temporary workaround for an issue in libstdc++ that has been fixed`
`186`	`186`	`* upstream [PR101510].`
	`187`	`+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510`
`187`	`188`	`*/`
`188`	`189`	`static inline bool create_directories(const std::filesystem::path& p)`
`189`	`190`	`{`