BP: MB-41352: Prune audit log older than X seconds

If the configuration includes "audit_prune_age":<somenumber> the
audit daemon will remove old audit files as part of checking if
files should be rotated or not.

If the value is set to 0 (or not present in the configuration)
the audit daemon will not try to prune old audit logs.

The file age is determined from the files "last write timestamp",
and all files matching "hostname[*]-audit.log" will be subject
for removal

Change-Id: I34205e514e16beba0248e9b9ac6bfa3ba59a4195
Reviewed-on: https://review.couchbase.org/c/kv_engine/+/200038
Well-Formed: Restriction Checker
Tested-by: Trond Norbye <[email protected]>
Reviewed-by: Jim Walker <[email protected]>

Author: trondn

auditd/README.md

@@ -358,6 +358,9 @@ the following fields:
   one day.  Minimum is 15 minutes)
 * rotate_size - number of bytes written to the file before rotating to a new
   file
+* prune_age - (optional field) Prune all audit log files older than the
+  specified value (in seconds). (set to 0 (or remove) to disable the
+  functionality).
 * buffered - should buffered file IO be used or not
 * disabled - list of event ids (numbers) containing those events that are NOT
   to be outputted to the audit log.  This is depreciated in version 2 and has

auditd/src/audit.cc

@@ -411,9 +411,7 @@ void AuditImpl::consume_events() {
 
     while (!stop_audit_consumer) {
         if (filleventqueue.empty()) {
-            events_arrived.wait_for(
-                    lock,
-                    std::chrono::seconds(auditfile.get_seconds_to_rotation()));
+            events_arrived.wait_for(lock, auditfile.get_sleep_time());
             if (filleventqueue.empty()) {
                 // We timed out, so just rotate the files
                 if (auditfile.maybe_rotate_files()) {

auditd/src/auditconfig.cc

@@ -56,10 +56,18 @@ AuditConfig::AuditConfig(const nlohmann::json& json) {
         }
     }
 
+    if (json.contains("prune_age")) {
+        auto val = json["prune_age"].get<size_t>();
+        if (val != 0) {
+            *(prune_age.lock()) = std::chrono::seconds(val);
+        }
+    }
+
     std::map<std::string, int> tags;
     tags["version"] = 1;
     tags["rotate_size"] = 1;
     tags["rotate_interval"] = 1;
+    tags["prune_age"] = 1;
     tags["auditd_enabled"] = 1;
     tags["buffered"] = 1;
     tags["log_path"] = 1;
@@ -418,7 +426,7 @@ void AuditConfig::initialize_config(const nlohmann::json& json) {
     disabled_userids.swap(other.disabled_userids);
     event_states.swap(other.event_states);
     uuid.swap(other.uuid);
-
+    prune_age.swap(other.prune_age);
     version = other.version;
 }
 

auditd/src/auditconfig.h

@@ -13,8 +13,10 @@
 #include <nlohmann/json_fwd.hpp>
 #include <relaxed_atomic.h>
 #include <atomic>
+#include <chrono>
 #include <cinttypes>
 #include <mutex>
+#include <optional>
 #include <string>
 #include <unordered_map>
 #include <utility> // For std::pair
@@ -102,6 +104,10 @@ class AuditConfig {
      */
     std::unique_ptr<AuditEventFilter> createAuditEventFilter(uint64_t rev);
 
+    [[nodiscard]] std::optional<std::chrono::seconds> get_prune_age() const {
+        return *prune_age.lock();
+    }
+
 protected:
     void sanitize_path(std::string &path);
     void add_array(std::vector<uint32_t>& vec,
@@ -138,6 +144,9 @@ class AuditConfig {
             event_states;
     folly::Synchronized<std::string, std::mutex> uuid;
 
+    folly::Synchronized<std::optional<std::chrono::seconds>, std::mutex>
+            prune_age;
+
     const uint32_t min_file_rotation_time = 900; // 15 minutes
     const uint32_t max_file_rotation_time = 604800; // 1 week
     const size_t max_rotate_file_size = 500 * 1024 * 1024; // 500MB

auditd/src/auditfile.cc

@@ -10,6 +10,7 @@
  */
 #include "auditfile.h"
 
+#include <folly/ScopeGuard.h>
 #include <logger/logger.h>
 #include <memcached/isotime.h>
 #include <nlohmann/json.hpp>
@@ -18,14 +19,78 @@
 #include <platform/platform_time.h>
 #include <platform/strerror.h>
 #include <utilities/json_utilities.h>
-
-#include <sys/stat.h>
 #include <algorithm>
-#include <cstring>
 #include <iostream>
 #include <sstream>
 
+/// c++20 std::string::starts_with
+bool starts_with(std::string_view name, std::string_view prefix) {
+    return name.find(prefix) == 0;
+}
+
+/// c++20 std::string::ends_with
+bool ends_with(std::string_view name, std::string_view suffix) {
+    const auto idx = name.rfind(suffix);
+    if (idx == std::string_view::npos) {
+        return false;
+    }
+    return (idx + suffix.size()) == name.length();
+}
+
+void AuditFile::iterate_old_files(
+        const std::function<void(const std::filesystem::path&)>& callback) {
+    using namespace std::filesystem;
+    for (const auto& p : directory_iterator(log_directory)) {
+        try {
+            const auto& path = p.path();
+            if (starts_with(path.filename().generic_string(), hostname) &&
+                ends_with(path.generic_string(), "-audit.log")) {
+                callback(path);
+            }
+        } catch (const std::exception& e) {
+            LOG_WARNING(
+                    "AuditFile::iterate_old_files(): Exception occurred "
+                    "while inspecting \"{}\": {}",
+                    p.path().generic_string(),
+                    e.what());
+        }
+    }
+}
+
+void AuditFile::prune_old_audit_files() {
+    using namespace std::chrono;
+    using namespace std::filesystem;
+
+    const auto filesystem_now = file_time_type::clock::now();
+    const auto now = steady_clock::now();
+    if (!prune_age.has_value() || next_prune > now) {
+        return;
+    }
+
+    auto oldest = filesystem_now;
+
+    const auto then = filesystem_now - *prune_age;
+    iterate_old_files([this, then, &oldest](const auto& path) {
+        auto mtime = last_write_time(path);
+        if (mtime < then) {
+            remove(path);
+        } else if (mtime < oldest) {
+            oldest = mtime;
+        }
+    });
+
+    if (oldest == filesystem_now) {
+        next_prune = now + *prune_age;
+    } else {
+        auto age = duration_cast<seconds>(filesystem_now - oldest);
+
+        // set next prune to a second after the oldest file expire
+        next_prune = now + (*prune_age - age) + seconds(1);
+    }
+}
+
 bool AuditFile::maybe_rotate_files() {
+    auto prune = folly::makeGuard([this] { prune_old_audit_files(); });
     if (is_open() && time_to_rotate_log()) {
         if (is_empty()) {
             // Given the audit log is empty on rotation instead of
@@ -66,6 +131,20 @@ uint32_t AuditFile::get_seconds_to_rotation() const {
     }
 }
 
+std::chrono::seconds AuditFile::get_sleep_time() const {
+    using namespace std::chrono;
+
+    const auto rotation = seconds{get_seconds_to_rotation()};
+    if (!prune_age) {
+        return rotation;
+    }
+    const auto now = steady_clock::now();
+    if (next_prune <= now) {
+        return seconds{0};
+    }
+    return std::min(rotation, duration_cast<seconds>(next_prune - now));
+}
+
 bool AuditFile::time_to_rotate_log() const {
     if (rotate_interval) {
         cb_assert(open_time != 0);
@@ -267,6 +346,8 @@ void AuditFile::reconfigure(const AuditConfig &config) {
     set_log_directory(config.get_log_directory());
     max_log_size = config.get_rotate_size();
     buffered = config.is_buffered();
+    prune_age = config.get_prune_age();
+    next_prune = std::chrono::steady_clock::now();
 }
 
 bool AuditFile::flush() {

auditd/src/auditfile.h

@@ -13,10 +13,14 @@
 #include "auditconfig.h"
 
 #include <nlohmann/json_fwd.hpp>
+#include <chrono>
 #include <cinttypes>
 #include <cstdio>
 #include <ctime>
+#include <filesystem>
+#include <functional>
 #include <memory>
+#include <optional>
 #include <string>
 
 class AuditFile {
@@ -85,6 +89,10 @@ class AuditFile {
      */
     uint32_t get_seconds_to_rotation() const;
 
+    [[nodiscard]] std::chrono::seconds get_sleep_time() const;
+
+    void prune_old_audit_files();
+
 protected:
     bool open();
     bool time_to_rotate_log() const;
@@ -98,6 +106,15 @@ class AuditFile {
 
     static time_t auditd_time();
 
+    /**
+     * Iterate over "old" audit log files (named hostname-<timestamp>-audit.log)
+     * and call the provided callback with the path
+     *
+     * @param callback
+     */
+    void iterate_old_files(
+            const std::function<void(const std::filesystem::path&)>& callback);
+
     struct FileDeleter {
         void operator()(FILE* fp) {
             fclose(fp);
@@ -112,6 +129,13 @@ class AuditFile {
     size_t current_size = 0;
     size_t max_log_size = 20 * 1024 * 1024;
     uint32_t rotate_interval = 900;
+    std::optional<std::chrono::seconds> prune_age;
+    /// Iterating over all files in the directory and fetch their
+    /// modification time may be "costly" so we don't want to run
+    /// it too often.
+    std::chrono::steady_clock::time_point next_prune =
+            std::chrono::steady_clock::now();
+
     bool buffered = true;
 };
 

auditd/tests/auditconfig_test.cc

@@ -500,3 +500,15 @@ TEST_F(AuditConfigTest, TestSpecifyEventStates) {
         }
     }
 }
+
+TEST_F(AuditConfigTest, AuditPruneAge) {
+    config.initialize_config(json);
+    EXPECT_FALSE(config.get_prune_age().has_value());
+    json["prune_age"] = 0;
+    config.initialize_config(json);
+    EXPECT_FALSE(config.get_prune_age().has_value());
+    json["prune_age"] = 1000;
+    config.initialize_config(json);
+    EXPECT_TRUE(config.get_prune_age().has_value());
+    EXPECT_EQ(1000, config.get_prune_age().value().count());
+}

auditd/tests/auditfile_test.cc

@@ -11,12 +11,16 @@
 #include <platform/dirutils.h>
 
 #include "auditfile.h"
+#include <fmt/format.h>
 #include <folly/portability/GTest.h>
+#include <memcached/isotime.h>
 #include <nlohmann/json.hpp>
 #include <platform/platform_time.h>
+#include <platform/strerror.h>
 #include <time.h>
 #include <atomic>
 #include <cstring>
+#include <deque>
 #include <iostream>
 #include <map>
 
@@ -323,3 +327,87 @@ TEST_F(AuditFileTest, MB53282) {
     auditfile.reconfigure(config);
     auditfile.test_mb53282();
 }
+
+TEST_F(AuditFileTest, PruneFiles) {
+    class MockAuditFile : public AuditFile {
+    public:
+        explicit MockAuditFile(const std::filesystem::path& logdir)
+            : AuditFile("PruneFiles") {
+            set_log_directory(logdir.generic_string());
+            for (int ii = 0; ii < 10; ++ii) {
+                createAuditLogFile(
+                        logdir, "PruneFiles", std::chrono::hours(ii));
+            }
+        };
+
+        static void createAuditLogFile(const std::filesystem::path& logdir,
+                                       std::string_view hostname,
+                                       std::chrono::seconds seconds) {
+            using namespace std::filesystem;
+            auto ts = ISOTime::generatetimestamp(
+                              time(nullptr) - seconds.count(), 0)
+                              .substr(0, 19);
+            std::replace(ts.begin(), ts.end(), ':', '-');
+            auto filename = fmt::format("{}-{}-audit.log", hostname, ts);
+            auto path = logdir / fmt::format("{}-{}-audit.log", hostname, ts);
+            FILE* fp = fopen(path.generic_string().c_str(), "w");
+            if (!fp) {
+                throw std::runtime_error(fmt::format(
+                        "createAuditLogFile: Failed to create {}: {}",
+                        path.generic_string(),
+                        cb_strerror()));
+            }
+            fclose(fp);
+            auto ftime = file_time_type::clock::now() - seconds;
+            last_write_time(path, ftime);
+        }
+
+        void set_prune_age(std::chrono::seconds age) {
+            using namespace std::chrono;
+            prune_age = age;
+            next_prune = steady_clock::now() - seconds(1);
+        }
+
+        std::deque<std::filesystem::path> get_log_files() {
+            std::deque<std::filesystem::path> ret;
+            for (const auto& p :
+                 std::filesystem::directory_iterator(log_directory)) {
+                if (is_regular_file(p.path())) {
+                    ret.push_back(p.path());
+                }
+            }
+
+            std::sort(ret.begin(), ret.end());
+            return ret;
+        }
+    };
+
+    MockAuditFile auditfile(testdir);
+    auto blueprint = auditfile.get_log_files();
+    EXPECT_EQ(10, blueprint.size());
+
+    // We don't have a prune time, so all files should be there
+    auditfile.prune_old_audit_files();
+    EXPECT_EQ(blueprint, auditfile.get_log_files());
+
+    // If we set the prune time longer than all the files they should still
+    // be there
+    auditfile.set_prune_age(std::chrono::hours(9) + std::chrono::minutes{30});
+    auditfile.prune_old_audit_files();
+    EXPECT_EQ(blueprint, auditfile.get_log_files());
+
+    // set the prune time between the two last files
+    auditfile.set_prune_age(std::chrono::hours(8) + std::chrono::minutes{30});
+    auditfile.prune_old_audit_files();
+    blueprint.pop_front();
+    EXPECT_EQ(blueprint, auditfile.get_log_files());
+
+    // Verify that we nuke multiple old files by specifying a time
+    // only leaving one file
+    auditfile.set_prune_age(std::chrono::minutes{30});
+    auditfile.prune_old_audit_files();
+    while (blueprint.size() > 1) {
+        blueprint.pop_front();
+    }
+    EXPECT_EQ(blueprint, auditfile.get_log_files());
+}