Merge pull request #28692 from travisdowns/td-BLL-speedup

BadLogLines speed ups

Author: travisdowns

proto/redpanda/core/admin/internal/v1/debug.proto

@@ -43,6 +43,22 @@ message ThrowStructuredExceptionRequest {
 }
 message ThrowStructuredExceptionResponse {}
 
+enum LogLevel {
+    LOG_LEVEL_UNSPECIFIED = 0;
+    LOG_LEVEL_TRACE = 1;
+    LOG_LEVEL_DEBUG = 2;
+    LOG_LEVEL_INFO = 3;
+    LOG_LEVEL_WARN = 4;
+    LOG_LEVEL_ERROR = 5;
+}
+
+message LogMessageRequest {
+    string message = 1;
+    LogLevel level = 2;
+}
+
+message LogMessageResponse {}
+
 // The DebugService provides access to internal debugging information and debug
 // operations for the cluster or node.
 //
@@ -71,4 +87,9 @@ service DebugService {
             authz: SUPERUSER,
         };
     }
+    rpc LogMessage(LogMessageRequest) returns (LogMessageResponse) {
+        option (pbgen.rpc) = {
+            authz: SUPERUSER,
+        };
+    }
 }

src/v/redpanda/admin/services/internal/debug.cc

@@ -131,4 +131,35 @@ debug_service_impl::stop_stress_fiber(
     co_return proto::stop_stress_fiber_response{};
 }
 
+seastar::future<proto::admin::log_message_response>
+debug_service_impl::log_message(
+  serde::pb::rpc::context, proto::admin::log_message_request req) {
+    auto msg = req.get_message();
+    auto level = req.get_level();
+
+    using enum proto::admin::log_level;
+
+    ss::log_level ss_level = [=]() {
+        switch (level) {
+        case trace:
+            return ss::log_level::trace;
+        case debug:
+            return ss::log_level::debug;
+        case info:
+            return ss::log_level::info;
+        case warn:
+            return ss::log_level::warn;
+        case error:
+            return ss::log_level::error;
+        case unspecified:
+        default:
+            throw serde::pb::rpc::invalid_argument_exception(
+              "Invalid log level specified");
+        }
+    }();
+
+    log.log(ss_level, "{}", msg);
+    co_return proto::admin::log_message_response{};
+}
+
 } // namespace admin

src/v/redpanda/admin/services/internal/debug.h

@@ -45,6 +45,9 @@ class debug_service_impl : public proto::admin::debug_service {
       serde::pb::rpc::context,
       proto::admin::stop_stress_fiber_request) override;
 
+    seastar::future<proto::admin::log_message_response> log_message(
+      serde::pb::rpc::context, proto::admin::log_message_request) override;
+
 private:
     admin::proxy::client _client;
     ss::sharded<stress_fiber_manager>& _stress_fiber_manager;

tests/docker/ducktape-deps/tool-pkgs

@@ -4,7 +4,8 @@
 # tests, but not for building the early layers of the image themselves. They
 # are installed relatively late in the Dockerfile, after most of the heavy
 # "non mainline" layers have forked off.
-set -e
+set -euo pipefail
+
 apt-get update
 apt-get install -qq \
   bind9-dnsutils \
@@ -57,3 +58,22 @@ unset LD_LIBRARY_PATH
 exec /usr/bin/llvm-symbolizer-$LLVM_VERSION "\$@"
 EOF
 chmod +x /usr/local/bin/llvm-symbolizer
+
+###########
+# ripgrep #
+###########
+
+RG_VERSION=15.1.0
+RG_BASE_URL="https://github.com/BurntSushi/ripgrep/releases/download"
+if [ $(uname -m) = "aarch64" ]; then
+  #           ripgrep-  15.1.0     -aarch64-unknown-linux-gnu.tar.gz
+  RG_TARBALL="ripgrep-${RG_VERSION}-aarch64-unknown-linux-gnu.tar.gz"
+else
+  #           ripgrep-  15.1.0     -x86_64-unknown-linux-musl.tar.gz
+  RG_TARBALL="ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz"
+fi
+mkdir -p /opt/ripgrep
+RG_URL="${RG_BASE_URL}/${RG_VERSION}/${RG_TARBALL}"
+echo "Downloading ripgrep from ${RG_URL}"
+curl -sSL "$RG_URL" | tar -xz -C /opt/ripgrep --strip-components=1
+ln -sf /opt/ripgrep/rg /usr/local/bin/rg

tests/rptest/clients/admin/proto/redpanda/core/admin/internal/v1/debug_pb2.py

@@ -26,7 +26,7 @@
     RedpandaService,
     RedpandaServiceCloud,
 )
-from rptest.services.utils import BadLogLines, VersionAndLines
+from rptest.services.utils import BadLogLines, NodeToLines, VersionAndLines
 
 from ducktape.tests.test import TestContext
 
@@ -150,7 +150,7 @@ def raise_on_bad_log_lines(self, node: ClusterNode) -> None:
         def make_vl() -> VersionAndLines:
             return {"version": None, "lines": []}
 
-        bad_lines: dict[ClusterNode, VersionAndLines] = collections.defaultdict(make_vl)
+        bad_lines: NodeToLines = collections.defaultdict(make_vl)
         self.logger.info(f"Scanning node {node.account.hostname} log for errors...")
 
         for line in node.account.ssh_capture(
@@ -420,7 +420,7 @@ def raise_on_bad_log_lines(self, node: ClusterNode) -> None:
         def make_vl() -> VersionAndLines:
             return {"version": None, "lines": []}
 
-        bad_lines: dict[ClusterNode, VersionAndLines] = collections.defaultdict(make_vl)
+        bad_lines: NodeToLines = collections.defaultdict(make_vl)
         self.logger.info(f"Scanning node {node.account.hostname} log for errors...")
 
         for line in node.account.ssh_capture(

tests/rptest/clients/admin/proto/redpanda/core/admin/internal/v1/debug_pb2.pyi

@@ -1294,7 +1294,7 @@ def all_up(self) -> bool:
 
     @abstractmethod
     def raise_on_bad_logs(
-        self, allow_list: LogAllowList = (), test_start_time: float | None = None
+        self, allow_list: LogAllowList = (), test_start_time: float = 0
     ) -> None:
         pass
 
@@ -2340,17 +2340,21 @@ def _get_restart_count(p: dict[str, Any]):
             # Check if stored pod and loaded one is the same
             _stored_pod = _get_stored_pod(pod["metadata"]["uid"])
             if _stored_pod is None:
-                raise NodeCrash((_name, "Pod not found among prior stored ones"))
+                raise NodeCrash([(_name, "Pod not found among prior stored ones")])
 
             # Check if container inside pod stayed the same
             container_id = _get_container_id(pod["status"])
             if _get_container_id(_stored_pod._status) != container_id:
-                raise NodeCrash((_name, "Pod container mismatch with prior stored one"))
+                raise NodeCrash(
+                    [(_name, "Pod container mismatch with prior stored one")]
+                )
 
             # Check that restart count is the same
             restart_count = _get_restart_count(pod["status"])
             if _get_restart_count(_stored_pod._status) != restart_count:
-                raise NodeCrash((_name, "Pod has been restarted due to possible crash"))
+                raise NodeCrash(
+                    [(_name, "Pod has been restarted due to possible crash")]
+                )
 
         # Worth to note that rebuilding stored broker classes
         # can be skipped in this case since nothing changed now
@@ -2397,7 +2401,7 @@ def cluster_healthy(self) -> bool:
         return self.cluster_unhealthy_reason is not None
 
     def raise_on_bad_logs(
-        self, allow_list: LogAllowList = (), test_start_time: float | None = None
+        self, allow_list: LogAllowList = (), test_start_time: float = 0
     ) -> None:
         """
         Raise a BadLogLines exception if any nodes' logs contain errors
@@ -3020,7 +3024,7 @@ def set_skip_if_no_redpanda_log(self, v: bool):
         self._skip_if_no_redpanda_log = v
 
     def raise_on_bad_logs(
-        self, allow_list: LogAllowList = (), test_start_time: float | None = None
+        self, allow_list: LogAllowList = (), test_start_time: float = 0
     ):
         """
         Raise a BadLogLines exception if any nodes' logs contain errors not