Merge pull request #2051 from jan-cerny/items_limit

Introduce a limit of collected items

Author: evgenyz

docs/manual/manual.adoc

@@ -1618,6 +1618,7 @@ not considered local by the scanner:
 * `SEXP_VALIDATE_DISABLE` - If set, `oscap` will not validate SEXP expressions during its execution.
 * `SOURCE_DATE_EPOCH` - Timestamp in seconds since epoch. This timestamp will be used instead of the current time to populate `timestamp` attributes in SCAP source data streams created by `oscap ds sds-compose` sub-module. This is used for reproducible builds of data streams.
 * `OSCAP_PROBE_MEMORY_USAGE_RATIO` - maximum memory usage ratio (used/total) for OpenSCAP probes, default: 0.1
+* `OSCAP_PROBE_MAX_COLLECTED_ITEMS` - maximal count of collected items by OpenSCAP probe for a single OVAL object evaluation
 
 Also, OpenSCAP uses `libcurl` library which also can be configured using environment variables. See https://curl.se/libcurl/c/libcurl-env.html[the list of libcurl environment variables].
 

src/OVAL/probes/independent/textfilecontent54_probe.c

@@ -240,11 +240,14 @@ static int process_file(const char *prefix, const char *path, const char *file,
 				item = create_item(path, file, pfd->pattern,
 						cur_inst, substrs, substr_cnt, over);
 
-                                probe_item_collect(pfd->ctx, item);
-
 				for (k = 0; k < substr_cnt; ++k)
 					free(substrs[k]);
 				free(substrs);
+				int pic_ret = probe_item_collect(pfd->ctx, item);
+				if (pic_ret == 2 || pic_ret == -1) {
+					ret = -4;
+					break;
+				}
 			}
 		}
 	} while (substr_cnt > 0 && ofs < buf_used);

src/OVAL/probes/probe/icache.c

@@ -39,6 +39,7 @@
 #include "probe-api.h"
 #include "common/debug_priv.h"
 #include "common/memusage.h"
+#include "oscap_helpers.h"
 
 #include "probe.h"
 #include "icache.h"
@@ -528,6 +529,30 @@ static int probe_cobj_memcheck(size_t item_cnt, double max_ratio)
 	return (0);
 }
 
+static int _mark_collected_object_as_incomplete(struct probe_ctx *ctx, const char *message)
+{
+	/*
+	 * Don't set the message again if the collected object is
+	 * already flagged as incomplete.
+	 */
+	if (probe_cobj_get_flag(ctx->probe_out) == SYSCHAR_FLAG_INCOMPLETE) {
+		return 0;
+	}
+	/*
+	 * Sync with the icache thread before modifying the
+	 * collected object.
+	 */
+	if (probe_icache_nop(ctx->icache) != 0) {
+		return -1;
+	}
+
+	SEXP_t *sexp_msg = probe_msg_creat(OVAL_MESSAGE_LEVEL_WARNING, (char *) message);
+	probe_cobj_add_msg(ctx->probe_out, sexp_msg);
+	probe_cobj_set_flag(ctx->probe_out, SYSCHAR_FLAG_INCOMPLETE);
+	SEXP_free(sexp_msg);
+	return 0;
+}
+
 /**
  * Collect an item
  * This function adds an item the collected object assosiated
@@ -557,6 +582,16 @@ int probe_item_collect(struct probe_ctx *ctx, SEXP_t *item)
 	cobj_itemcnt = SEXP_list_length(cobj_content);
 	SEXP_free(cobj_content);
 
+	if (ctx->max_collected_items != OSCAP_PROBE_COLLECT_UNLIMITED && cobj_itemcnt >= ctx->max_collected_items) {
+		char *message = oscap_sprintf("Object is incomplete because the object matches more than %ld items.", ctx->max_collected_items);
+		if (_mark_collected_object_as_incomplete(ctx, message) != 0) {
+			free(message);
+			return -1;
+		}
+		free(message);
+		return 2;
+	}
+
 	memcheck_ret = probe_cobj_memcheck(cobj_itemcnt, ctx->max_mem_ratio);
 	if (memcheck_ret == -1) {
 		dE("Failed to check available memory");
@@ -565,27 +600,9 @@ int probe_item_collect(struct probe_ctx *ctx, SEXP_t *item)
 	}
 	if (memcheck_ret == 1) {
 		SEXP_free(item);
-
-		/*
-		 * Don't set the message again if the collected object is
-		 * already flagged as incomplete.
-		 */
-		if (probe_cobj_get_flag(ctx->probe_out) != SYSCHAR_FLAG_INCOMPLETE) {
-			SEXP_t *msg;
-			/*
-			 * Sync with the icache thread before modifying the
-			 * collected object.
-			 */
-			if (probe_icache_nop(ctx->icache) != 0)
-				return -1;
-
-			msg = probe_msg_creat(OVAL_MESSAGE_LEVEL_WARNING,
-			                      "Object is incomplete due to memory constraints.");
-
-			probe_cobj_add_msg(ctx->probe_out, msg);
-			probe_cobj_set_flag(ctx->probe_out, SYSCHAR_FLAG_INCOMPLETE);
-
-			SEXP_free(msg);
+		const char *message = "Object is incomplete due to memory constraints.";
+		if (_mark_collected_object_as_incomplete(ctx, message) != 0) {
+			return -1;
 		}
 
 		return 2;

src/OVAL/probes/probe/probe.h

@@ -43,6 +43,15 @@
 #include "common/util.h"
 #include "common/compat_pthread_barrier.h"
 
+/* default max. memory usage ratio - used/total */
+/* can be overridden by environment variable OSCAP_PROBE_MEMORY_USAGE_RATIO */
+#define OSCAP_PROBE_MEMORY_USAGE_RATIO_DEFAULT 0.33
+
+/* By default, probes can collect unlimited amount of items. Ths behavior can
+ * be overridden by environment variable OSCAP_PROBE_MAX_COLLECTED_ITEMS.
+ */
+#define OSCAP_PROBE_COLLECT_UNLIMITED 0
+
 typedef struct {
 	pthread_rwlock_t rwlock;
 	uint32_t         flags;
@@ -84,6 +93,7 @@ struct probe_ctx {
         probe_icache_t *icache;    /**< item cache */
 	int offline_mode;
 	double max_mem_ratio;
+	size_t max_collected_items;
 };
 
 typedef enum {

src/OVAL/probes/probe/worker.c

@@ -52,10 +52,6 @@ extern int chroot(const char *);
 #include "probe-table.h"
 #include "probe.h"
 
-/* default max. memory usage ratio - used/total */
-/* can be overridden by environment variable OSCAP_PROBE_MEMORY_USAGE_RATIO */
-#define OSCAP_PROBE_MEMORY_USAGE_RATIO_DEFAULT 0.33
-
 extern bool  OSCAP_GSYM(varref_handling);
 extern void *OSCAP_GSYM(probe_arg);
 
@@ -1078,6 +1074,14 @@ SEXP_t *probe_worker(probe_t *probe, SEAP_msg_t *msg_in, int *ret)
 			if (max_ratio > 0)
 				pctx.max_mem_ratio = max_ratio;
 		}
+		pctx.max_collected_items = OSCAP_PROBE_COLLECT_UNLIMITED;
+		char *max_collected_items_str = getenv("OSCAP_PROBE_MAX_COLLECTED_ITEMS");
+		if (max_collected_items_str != NULL) {
+			int max_collected_items = strtol(max_collected_items_str, NULL, 0);
+			if (max_collected_items > 0) {
+				pctx.max_collected_items = max_collected_items;
+			}
+		}
 
 		/* simple object */
                 pctx.icache  = probe->icache;

tests/CMakeLists.txt

@@ -33,6 +33,7 @@ add_subdirectory("mitre")
 add_subdirectory("nist")
 add_subdirectory("oscap_string")
 add_subdirectory("oval_details")
+add_subdirectory("probe_behavior")
 add_subdirectory("probes")
 add_subdirectory("report")
 add_subdirectory("sce")

tests/probe_behavior/CMakeLists.txt

@@ -0,0 +1 @@
+add_oscap_test("collect_limit.sh")

tests/memory/cgroup_test.sh renamed to tests/probe_behavior/cgroup_test.sh

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<oval_definitions xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" xmlns:ind="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ind-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:unix-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#unix" xmlns:lin-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#linux" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5" xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5#unix unix-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#linux linux-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-common-5 oval-common-schema.xsd">
+  <generator>
+    <oval:schema_version>5.10.1</oval:schema_version>
+    <oval:timestamp>0001-01-01T00:00:00+00:00</oval:timestamp>
+  </generator>
+
+  <definitions>
+    <definition class="compliance" version="1" id="oval:x:def:1">
+      <metadata>
+        <title>x</title>
+        <description>x</description>
+        <affected family="unix">
+          <platform>x</platform>
+        </affected>
+      </metadata>
+      <criteria>
+        <criterion test_ref="oval:x:tst:1" comment="always pass"/>
+      </criteria>
+    </definition>
+  </definitions>
+
+  <tests>
+    <ind:textfilecontent54_test id="oval:x:tst:1" version="1" comment="Test" check="all">
+      <ind:object object_ref="oval:x:obj:1"/>
+    </ind:textfilecontent54_test>
+  </tests>
+
+  <objects>
+    <ind:textfilecontent54_object id="oval:x:obj:1" version="1" comment="The object matches each line in /tmp/longfile.">
+      <ind:filepath>/tmp/longfile</ind:filepath>
+      <ind:pattern operation="pattern match">^.*$</ind:pattern>
+      <ind:instance datatype="int" operation="greater than or equal">1</ind:instance>
+    </ind:textfilecontent54_object>
+  </objects>
+
+</oval_definitions>

tests/probe_behavior/collect_limit.oval.xml

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+. $builddir/tests/test_common.sh
+
+export OSCAP_PROBE_MAX_COLLECTED_ITEMS=100
+seq 110 > /tmp/longfile
+result=$(mktemp)
+$OSCAP oval eval --results "$result" $srcdir/collect_limit.oval.xml
+assert_exists 1 '/oval_results/results/system/oval_system_characteristics/collected_objects/object'
+assert_exists 1 '/oval_results/results/system/oval_system_characteristics/collected_objects/object[@flag="incomplete"]'
+assert_exists 1 '/oval_results/results/system/oval_system_characteristics/collected_objects/object/message[@level="warning"]'
+text="Object is incomplete because the object matches more than 100 items."
+assert_exists 1 "/oval_results/results/system/oval_system_characteristics/collected_objects/object/message[text()=\"$text\"]"
+assert_exists 100 '/oval_results/results/system/oval_system_characteristics/system_data/ind-sys:textfilecontent_item'
+rm -f /tmp/longfile
+rm -f "$result"