Add a backoff cache for unactionable stored events (#623)

Author: mlw

Source/common/SNTStoredEvent.h

@@ -28,4 +28,8 @@
 // The base class implementation will throw an exception.
 - (nullable NSString *)uniqueID;
 
+// Subclasses are required to define: `unactionableEvent`.
+// The base class implementation will throw an exception.
+- (BOOL)unactionableEvent;
+
 @end

Source/common/SNTStoredEvent.mm

@@ -65,4 +65,9 @@ - (NSString *)uniqueID {
   return nil;
 }
 
+- (BOOL)unactionableEvent {
+  [self doesNotRecognizeSelector:_cmd];
+  return NO;
+}
+
 @end

Source/common/SNTStoredEventTest.mm

@@ -42,6 +42,26 @@ - (void)testUniqueID {
   XCTAssertEqualObjects([faaEvent uniqueID], @"MyRule|MyVersion|bar");
 }
 
+- (void)testUnactionableEvent {
+  // The base class should throw
+  SNTStoredEvent *baseEvent = [[SNTStoredEvent alloc] init];
+  XCTAssertThrows([baseEvent unactionableEvent]);
+
+  // Spot check allow/block events
+  SNTStoredExecutionEvent *execEvent = [[SNTStoredExecutionEvent alloc] init];
+  execEvent.decision = SNTEventStateAllowBinary;
+  XCTAssertTrue([execEvent unactionableEvent]);
+  execEvent.decision = SNTEventStateBlockBinary;
+  XCTAssertFalse([execEvent unactionableEvent]);
+
+  // Spot check audit only/denied events
+  SNTStoredFileAccessEvent *faaEvent = [[SNTStoredFileAccessEvent alloc] init];
+  faaEvent.decision = FileAccessPolicyDecision::kAllowedAuditOnly;
+  XCTAssertTrue([faaEvent unactionableEvent]);
+  faaEvent.decision = FileAccessPolicyDecision::kDenied;
+  XCTAssertFalse([faaEvent unactionableEvent]);
+}
+
 - (void)testEncodeDecode {
   SNTStoredExecutionEvent *execEvent = [[SNTStoredExecutionEvent alloc] init];
   execEvent.fileSHA256 = @"foo";

Source/common/SNTStoredExecutionEvent.mm

@@ -171,4 +171,8 @@ - (NSString *)uniqueID {
   return self.fileSHA256;
 }
 
+- (BOOL)unactionableEvent {
+  return (self.decision & SNTEventStateAllow) != 0;
+}
+
 @end

Source/common/SNTStoredFileAccessEvent.mm

@@ -62,6 +62,10 @@ - (NSString *)uniqueID {
   return [NSString stringWithFormat:@"%@|%@|%@", _ruleName, _ruleVersion, _process.fileSHA256];
 }
 
+- (BOOL)unactionableEvent {
+  return self.decision == FileAccessPolicyDecision::kAllowedAuditOnly;
+}
+
 @end
 
 @implementation SNTStoredFileAccessProcess

Source/santad/BUILD

@@ -57,6 +57,8 @@ objc_library(
         "//Source/common:SNTLogging",
         "//Source/common:SNTStoredExecutionEvent",
         "//Source/common:SNTStoredFileAccessEvent",
+        "//Source/common:SantaCache",
+        "//Source/common:String",
     ],
 )
 
@@ -1061,6 +1063,9 @@ santa_unit_test(
         "//Source/common:SNTLogging",
         "//Source/common:SNTStoredExecutionEvent",
         "//Source/common:SNTStoredFileAccessEvent",
+        "//Source/common:SantaCache",
+        "//Source/common:String",
+        "//Source/common:TestUtils",
         "@FMDB",
         "@OCMock",
     ],

Source/santad/DataLayer/SNTEventTable.mm

@@ -15,21 +15,37 @@
 
 #import "Source/santad/DataLayer/SNTEventTable.h"
 
+#include <memory>
+
 #import "Source/common/MOLCertificate.h"
 #import "Source/common/SNTLogging.h"
 #import "Source/common/SNTStoredEvent.h"
 #import "Source/common/SNTStoredExecutionEvent.h"
 #import "Source/common/SNTStoredFileAccessEvent.h"
+#include "Source/common/SantaCache.h"
+#include "Source/common/String.h"
 
 static const uint32_t kEventTableCurrentVersion = 5;
+// 4 hour cache
+static const NSTimeInterval kUnactionableEventCacheTimeSeconds = (60 * 60 * 4);
+
+@interface SNTEventTable ()
+// This property is only set once, safe to be nonatomic
+@property(nonatomic) NSTimeInterval unactionableEventCacheTimeSeconds;
+@end
 
-@implementation SNTEventTable
+@implementation SNTEventTable {
+  std::unique_ptr<SantaCache<std::string, NSDate *>> _storeBackoff;
+}
 
 - (uint32_t)currentSupportedVersion {
   return kEventTableCurrentVersion;
 }
 
 - (uint32_t)initializeDatabase:(FMDatabase *)db fromVersion:(uint32_t)version {
+  _storeBackoff = std::make_unique<SantaCache<std::string, NSDate *>>(2048);
+  self.unactionableEventCacheTimeSeconds = kUnactionableEventCacheTimeSeconds;
+
   int newVersion = 0;
 
   if (version < 1) {
@@ -112,6 +128,10 @@ - (BOOL)addStoredEvents:(NSArray<SNTStoredEvent *> *)events {
       continue;
     }
 
+    if ([event unactionableEvent] && [self backoffForPrimaryHash:[event uniqueID]]) {
+      continue;
+    }
+
     NSData *eventData = [NSKeyedArchiver archivedDataWithRootObject:event
                                               requiringSecureCoding:YES
                                                               error:nil];
@@ -135,6 +155,18 @@ - (BOOL)addStoredEvents:(NSArray<SNTStoredEvent *> *)events {
   return success;
 }
 
+- (BOOL)backoffForPrimaryHash:(NSString *)hash {
+  NSDate *backoff = _storeBackoff->get(santa::NSStringToUTF8String(hash));
+  NSDate *now = [NSDate date];
+  if (([now timeIntervalSince1970] - [backoff timeIntervalSince1970]) <
+      self.unactionableEventCacheTimeSeconds) {
+    return YES;
+  } else {
+    _storeBackoff->set(santa::NSStringToUTF8String(hash), now);
+    return NO;
+  }
+}
+
 #pragma mark Querying/Retreiving
 
 - (NSUInteger)pendingEventsCount {

Source/santad/DataLayer/SNTEventTableTest.mm

@@ -24,6 +24,7 @@
 #import "Source/common/SNTFileInfo.h"
 #import "Source/common/SNTStoredExecutionEvent.h"
 #import "Source/common/SNTStoredFileAccessEvent.h"
+#include "Source/common/TestUtils.h"
 
 NSString *GenerateRandomHexStringWithSHA256Length() {
   // Create an array to hold random bytes
@@ -48,6 +49,7 @@
 
 @interface SNTEventTable (Testing)
 - (SNTStoredEvent *)eventFromResultSet:(FMResultSet *)rs;
+@property NSTimeInterval unactionableEventCacheTimeSeconds;
 @end
 
 /// This test case actually tests SNTEventTable and SNTStoredEvent.
@@ -82,7 +84,7 @@ - (SNTStoredExecutionEvent *)createTestEvent {
   event.executingUser = @"nobody";
   event.loggedInUsers = @[ @"nobody" ];
   event.currentSessions = @[ @"nobody@ttys000", @"nobody@console" ];
-  event.decision = SNTEventStateAllowBinary;
+  event.decision = SNTEventStateBlockBinary;
   return event;
 }
 
@@ -100,6 +102,7 @@ - (SNTStoredFileAccessEvent *)createTestFileAccessEvent {
   event.process.pid = @(1234);
   event.process.parent = [[SNTStoredFileAccessProcess alloc] init];
   event.process.parent.pid = @(4567);
+  event.decision = FileAccessPolicyDecision::kDenied;
   return event;
 }
 
@@ -111,7 +114,7 @@ - (void)testAddEvent {
   XCTAssertEqual(self.sut.pendingEventsCount, 2);
 }
 
-- (void)testUniqueIndex {
+- (void)testUniqueIndexActionable {
   XCTAssertEqual(self.sut.pendingEventsCount, 0);
 
   SNTStoredExecutionEvent *event = [self createTestEvent];
@@ -157,6 +160,94 @@ - (void)testUniqueIndex {
   XCTAssertEqual(self.sut.pendingEventsCount, 4);
 }
 
+- (void)testUniqueIndexUnactionable {
+  XCTAssertEqual(self.sut.pendingEventsCount, 0);
+
+  SNTStoredExecutionEvent *event = [self createTestEvent];
+  // Make this an "unactionable" event
+  event.decision = SNTEventStateAllowBinary;
+  XCTAssertTrue([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 1);
+
+  // Attempt to add an event with the same file hash will fail because
+  // no insert will be attempted
+  event.idx = @(arc4random());
+  XCTAssertFalse([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 1);
+
+  // Create a new hash and re-insert
+  event.idx = @(arc4random());
+  event.fileSHA256 = GenerateRandomHexStringWithSHA256Length();
+  XCTAssertTrue([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 2);
+
+  // Attempting to add an event with a non-unique idx fails
+  event.fileSHA256 = GenerateRandomHexStringWithSHA256Length();
+  XCTAssertFalse([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 2);
+
+  // Now for FAA Events...
+  SNTStoredFileAccessEvent *faaEvent = [self createTestFileAccessEvent];
+  // Make this an "unactionable" event
+  faaEvent.decision = FileAccessPolicyDecision::kAllowedAuditOnly;
+  XCTAssertTrue([self.sut addStoredEvent:faaEvent]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 3);
+
+  // Attempt to add an event with the same file hash will fail because
+  // no insert will be attempted
+  faaEvent.idx = @(arc4random());
+  XCTAssertFalse([self.sut addStoredEvent:faaEvent]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 3);
+
+  // Create a new hash and re-insert
+  faaEvent.process.fileSHA256 = GenerateRandomHexStringWithSHA256Length();
+  XCTAssertTrue([self.sut addStoredEvent:faaEvent]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 4);
+
+  // Attempting to add an event with a non-unique idx fails
+  faaEvent.process.fileSHA256 = GenerateRandomHexStringWithSHA256Length();
+  XCTAssertFalse([self.sut addStoredEvent:faaEvent]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 4);
+}
+
+- (void)testBackoff {
+  XCTAssertEqual(self.sut.pendingEventsCount, 0);
+
+  // Set the backoff time to 3 seconds
+  self.sut.unactionableEventCacheTimeSeconds = 3;
+
+  SNTStoredExecutionEvent *event = [self createTestEvent];
+  // Make this an "unactionable" event
+  event.decision = SNTEventStateAllowBinary;
+  NSNumber *origId = event.idx;
+  XCTAssertTrue([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 1);
+
+  // Attempt to add an event with the same file hash will fail because
+  // no insert will be attempted
+  event.idx = @(arc4random());
+  XCTAssertFalse([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 1);
+
+  // Delete everything
+  [self.sut deleteEventWithId:origId];
+  XCTAssertEqual(self.sut.pendingEventsCount, 0);
+
+  // Attempt to reinsert an unactionable event before the backoff time
+  // This should still fail, even though it wouldn't conflict in the DB
+  event.idx = @(arc4random());
+  XCTAssertFalse([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 0);
+
+  // Sleep for the backoff time
+  SleepMS(3000);
+
+  // Now re-add the stored event, all should be well
+  event.idx = @(arc4random());
+  XCTAssertTrue([self.sut addStoredEvent:event]);
+  XCTAssertEqual(self.sut.pendingEventsCount, 1);
+}
+
 - (void)testRetrieveExecutionEvent {
   SNTStoredExecutionEvent *event = [self createTestEvent];
   [self.sut addStoredEvent:event];

Source/santad/SNTSyncdQueue.h

@@ -23,6 +23,9 @@
 
 @interface SNTSyncdQueue : NSObject
 
+- (instancetype)initWithCacheSize:(uint64_t)cacheSize;
+- (instancetype)init NS_UNAVAILABLE;
+
 - (void)reassessSyncServiceConnection;
 - (void)reassessSyncServiceConnectionImmediately;
 

Source/santad/SNTSyncdQueue.mm

@@ -39,10 +39,10 @@ @implementation SNTSyncdQueue {
   std::unique_ptr<SantaCache<std::string, NSDate *>> _uploadBackoff;
 }
 
-- (instancetype)init {
+- (instancetype)initWithCacheSize:(uint64_t)cacheSize {
   self = [super init];
   if (self) {
-    _uploadBackoff = std::make_unique<SantaCache<std::string, NSDate *>>(256);
+    _uploadBackoff = std::make_unique<SantaCache<std::string, NSDate *>>(cacheSize);
     _syncdQueue = dispatch_queue_create("com.northpolesec.syncd_queue",
                                         DISPATCH_QUEUE_SERIAL_WITH_AUTORELEASE_POOL);
   }