Use raw events for regular Osquery machine snapshot updates

Author: np5

tests/osquery/test_osquery_api.py

@@ -10,6 +10,7 @@
 from zentral.conf import settings
 from zentral.contrib.inventory.events import MachineTagEvent
 from zentral.contrib.inventory.models import EnrollmentSecret, MachineSnapshot, MachineTag, MetaBusinessUnit, Tag
+from zentral.contrib.inventory.utils import commit_machine_snapshot_and_trigger_events
 from zentral.contrib.osquery.compliance_checks import sync_query_compliance_check
 from zentral.contrib.osquery.conf import INVENTORY_QUERY_NAME
 from zentral.contrib.osquery.events import (OsqueryEnrollmentEvent, OsqueryRequestEvent, OsqueryResultEvent,
@@ -273,24 +274,29 @@ def post_default_inventory_query_snapshot(
         missing_windows_build_data=False,
         unknown_windows_build=False,
     ):
-        return self.post_as_json(
-            "log",
-            {"node_key": node_key,
-             "log_type": "result",
-             "data": [{
-                 'action': 'snapshot',
-                 "name": INVENTORY_QUERY_NAME,
-                 "snapshot": self.get_default_inventory_query_snapshot(
-                     platform,
-                     with_app,
-                     with_ec2,
-                     no_windows_build_data,
-                     missing_windows_build_data,
-                     unknown_windows_build,
-                 ),
-                 'unixTime': '1480605737',
-             }]}
-        )
+        with patch("zentral.contrib.osquery.public_views.post_machine_snapshot_raw_event") as pmsre:
+            def store_mstree(ms_tree):
+                # simulate what is done by the preprocessor
+                commit_machine_snapshot_and_trigger_events(ms_tree)
+            pmsre.side_effect = store_mstree
+            return self.post_as_json(
+                "log",
+                {"node_key": node_key,
+                 "log_type": "result",
+                 "data": [{
+                     'action': 'snapshot',
+                     "name": INVENTORY_QUERY_NAME,
+                     "snapshot": self.get_default_inventory_query_snapshot(
+                         platform,
+                         with_app,
+                         with_ec2,
+                         no_windows_build_data,
+                         missing_windows_build_data,
+                         unknown_windows_build,
+                     ),
+                     'unixTime': '1480605737',
+                 }]}
+            )
 
     # enrollment
 

zentral/contrib/osquery/public_views.py

@@ -10,6 +10,7 @@
 from django.http import Http404, JsonResponse
 from django.utils.crypto import get_random_string
 from django.views.generic import View
+from zentral.contrib.inventory.events import post_machine_snapshot_raw_event
 from zentral.contrib.inventory.exceptions import EnrollmentSecretVerificationFailed
 from zentral.contrib.inventory.models import MachineSnapshot, MetaMachine
 from zentral.contrib.inventory.utils import (add_machine_tags,
@@ -144,6 +145,7 @@ def do_post(self):
             if business_unit:
                 tree["business_unit"] = business_unit.serialize()
             update_tree_with_enrollment_host_details(tree, self.data.get("host_details"))
+            # commit to push the extra inventory queries quickly
             commit_machine_snapshot_and_trigger_events(tree)
 
         post_enrollment_event(self.serial_number,
@@ -484,7 +486,8 @@ def do_node_post(self):
                 if business_unit:
                     tree["business_unit"] = business_unit.serialize()
                 update_tree_with_inventory_query_snapshot(tree, last_inventory_snapshot)
-                commit_machine_snapshot_and_trigger_events(tree)
+                # use the raw events queue to process this in the background
+                post_machine_snapshot_raw_event(tree)
             post_results(self.machine.serial_number, results, self.request)
         elif log_type == "status":
             # TODO: configuration option to filter some of those (severity) or maybe simply ignore them