Expand the storage protocol and provide an event iterator (#5742)

* Expand the storage protocol and provide an event iterator

* Remove unused variable

* Remove unused variable

* Make implicit self retain explicit

* Just check for NSFileTypeRegular instead

And fix a comment

* Missed a damned implicit self retain

* Implement some tests for the new API

* Fix yet another implicit self retain

Author: mikehaney24

GoogleDataTransport/GDTCORLibrary/GDTCORFlatFileStorage.m

@@ -20,10 +20,12 @@
 #import <GoogleDataTransport/GDTCORConsoleLogger.h>
 #import <GoogleDataTransport/GDTCOREvent.h>
 #import <GoogleDataTransport/GDTCORLifecycle.h>
+#import <GoogleDataTransport/GDTCORPlatform.h>
 #import <GoogleDataTransport/GDTCORPrioritizer.h>
 #import <GoogleDataTransport/GDTCORStorageEventSelector.h>
 
 #import "GDTCORLibrary/Private/GDTCOREvent_Private.h"
+#import "GDTCORLibrary/Private/GDTCORFlatFileStorageIterator.h"
 #import "GDTCORLibrary/Private/GDTCORRegistrar_Private.h"
 #import "GDTCORLibrary/Private/GDTCORUploadCoordinator.h"
 
@@ -323,7 +325,7 @@ - (void)libraryDataForKey:(nonnull NSString *)key
 
 - (void)storeLibraryData:(NSData *)data
                   forKey:(nonnull NSString *)key
-              onComplete:(nonnull void (^)(NSError *_Nullable error))onComplete {
+              onComplete:(nullable void (^)(NSError *_Nullable error))onComplete {
   if (!data || data.length <= 0) {
     if (onComplete) {
       onComplete([NSError errorWithDomain:NSInternalInconsistencyException code:-1 userInfo:nil]);
@@ -354,6 +356,76 @@ - (void)removeLibraryDataForKey:(nonnull NSString *)key
   });
 }
 
+- (BOOL)hasEventsForTarget:(GDTCORTarget)target {
+  NSString *searchPath = [GDTCORFlatFileStorage pathForTarget:target qosTier:nil mappingID:nil];
+  NSFileManager *fm = [NSFileManager defaultManager];
+  NSDirectoryEnumerator *enumerator = [fm enumeratorAtPath:searchPath];
+  while ([enumerator nextObject]) {
+    if ([enumerator.fileAttributes[NSFileType] isEqual:NSFileTypeRegular]) {
+      return YES;
+    }
+  }
+  return NO;
+}
+
+- (nullable id<GDTCORStorageEventIterator>)iteratorWithSelector:
+    (nonnull GDTCORStorageEventSelector *)eventSelector {
+  __block GDTCORFlatFileStorageIterator *iterator;
+  dispatch_sync(_storageQueue, ^{
+    NSMutableArray<NSString *> *filePaths;
+    NSArray<NSString *> *searchPaths =
+        [GDTCORFlatFileStorage searchPathsWithEventSelector:eventSelector];
+    for (NSString *searchPath in searchPaths) {
+      NSDirectoryEnumerator *enumerator =
+          [[NSFileManager defaultManager] enumeratorAtPath:searchPath];
+      NSString *nextFile;
+      while ((nextFile = [enumerator nextObject])) {
+        NSFileAttributeType fileType = enumerator.fileAttributes[NSFileType];
+        if ([fileType isEqual:NSFileTypeRegular]) {
+          [filePaths addObject:nextFile];
+        }
+      }
+      iterator = [[GDTCORFlatFileStorageIterator alloc] initWithTarget:eventSelector.selectedTarget
+                                                                 queue:self->_storageQueue];
+      iterator.eventFiles = filePaths;
+    }
+  });
+  return iterator;
+}
+
+- (void)purgeEventsFromBefore:(GDTCORClock *)beforeSnapshot
+                   onComplete:(void (^)(NSError *_Nullable error))onComplete {
+  // TODO(mikehaney24): Figure out how we're going to deal with this.
+}
+
+- (void)storageSizeWithCallback:(void (^)(uint64_t storageSize))onComplete {
+  dispatch_async(_storageQueue, ^{
+    unsigned long long totalBytes = 0;
+    NSString *eventDataPath = [GDTCORFlatFileStorage baseEventStoragePath];
+    NSString *libraryDataPath = [GDTCORFlatFileStorage libraryDataStoragePath];
+    NSDirectoryEnumerator *enumerator =
+        [[NSFileManager defaultManager] enumeratorAtPath:eventDataPath];
+    while ([enumerator nextObject]) {
+      NSFileAttributeType fileType = enumerator.fileAttributes[NSFileType];
+      if ([fileType isEqual:NSFileTypeRegular]) {
+        NSNumber *fileSize = enumerator.fileAttributes[NSFileSize];
+        totalBytes += fileSize.unsignedLongLongValue;
+      }
+    }
+    enumerator = [[NSFileManager defaultManager] enumeratorAtPath:libraryDataPath];
+    while ([enumerator nextObject]) {
+      NSFileAttributeType fileType = enumerator.fileAttributes[NSFileType];
+      if ([fileType isEqual:NSFileTypeRegular]) {
+        NSNumber *fileSize = enumerator.fileAttributes[NSFileSize];
+        totalBytes += fileSize.unsignedLongLongValue;
+      }
+    }
+    if (onComplete) {
+      onComplete(totalBytes);
+    }
+  });
+}
+
 #pragma mark - Private helper methods
 
 /** Saves the event's dataObject to a file using NSData mechanisms.

GoogleDataTransport/GDTCORLibrary/GDTCORFlatFileStorageIterator.m

@@ -0,0 +1,66 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import "GDTCORLibrary/Private/GDTCORFlatFileStorageIterator.h"
+
+#import <GoogleDataTransport/GDTCORConsoleLogger.h>
+#import <GoogleDataTransport/GDTCOREvent.h>
+
+@implementation GDTCORFlatFileStorageIterator {
+  /** The current eventFiles array index, incremented in -nextEvent. */
+  NSInteger _currentIndex;
+}
+
+- (instancetype)initWithTarget:(GDTCORTarget)target queue:(dispatch_queue_t)queue {
+  self = [super init];
+  if (self) {
+    _queue = queue;
+    _target = target;
+  }
+  return self;
+}
+
+- (nullable GDTCOREvent *)nextEvent {
+  if (_currentIndex == -1) {
+    return nil;
+  }
+  if (!_eventFiles) {
+    GDTCORLogDebug(@"%@", @"eventFiles property not set, so -nextEvent will be nil.");
+    return nil;
+  }
+  dispatch_queue_t queue = _queue;
+  if (!queue) {
+    GDTCORLogDebug(@"%@", @"iterator queue was nil, so -nextEvent will be nil.");
+    return nil;
+  }
+  __block GDTCOREvent *nextEvent;
+  dispatch_sync(queue, ^{
+    NSData *data = [NSData dataWithContentsOfFile:self->_eventFiles[self->_currentIndex]];
+    if (data) {
+      NSError *error;
+      nextEvent = (GDTCOREvent *)GDTCORDecodeArchive([GDTCOREvent class], nil, data, &error);
+      if (error || nextEvent == nil) {
+        GDTCORLogDebug(@"Unarchiving an event failed: %@", error);
+        nextEvent = nil;
+        self->_currentIndex = -1;
+        return;
+      }
+    }
+  });
+  return nextEvent;
+}
+
+@end

GoogleDataTransport/GDTCORLibrary/Private/GDTCORFlatFileStorageIterator.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#import <Foundation/Foundation.h>
+
+#import <GoogleDataTransport/GDTCORStorageProtocol.h>
+
+/** The flat file storage iterator. */
+@interface GDTCORFlatFileStorageIterator : NSObject <GDTCORStorageEventIterator>
+
+/** The queue on which the iterator will run. */
+@property(nonatomic, readonly) dispatch_queue_t queue;
+
+/** The target the iterator will work on. */
+@property(nonatomic, readonly) GDTCORTarget target;
+
+/** The list of event files to iterate over. This should be set before calling -nextEvent. */
+@property(nonatomic) NSArray<NSString *> *eventFiles;
+
+@end

GoogleDataTransport/GDTCORLibrary/Public/GDTCORStorageProtocol.h

@@ -17,12 +17,35 @@
 #import <Foundation/Foundation.h>
 
 #import <GoogleDataTransport/GDTCORLifecycle.h>
+#import <GoogleDataTransport/GDTCORStorageEventSelector.h>
 #import <GoogleDataTransport/GDTCORTargets.h>
 
 @class GDTCOREvent;
+@class GDTCORClock;
 
 NS_ASSUME_NONNULL_BEGIN
 
+/** Defines an iterator API for processing or fetching events. */
+@protocol GDTCORStorageEventIterator <NSObject>
+
+@required
+
+/** Initializes an iterator instance with the given target and queue.
+ *
+ * @param target The target for which the events are stored.
+ * @param queue The queue on which the iterator should run.
+ * @return An iterator instance.
+ */
+- (instancetype)initWithTarget:(GDTCORTarget)target queue:(dispatch_queue_t)queue;
+
+/** Returns the next event or nil if there are no more events to iterate over.
+ *
+ * @return A GDTCOREvent instance or nil if the iterator has iterated through all the events.
+ */
+- (nullable GDTCOREvent *)nextEvent;
+
+@end
+
 /** Defines the interface a storage subsystem is expected to implement. */
 @protocol GDTCORStorageProtocol <NSObject, GDTCORLifecycleProtocol>
 
@@ -47,7 +70,7 @@ NS_ASSUME_NONNULL_BEGIN
  */
 - (void)storeLibraryData:(NSData *)data
                   forKey:(NSString *)key
-              onComplete:(void (^)(NSError *_Nullable error))onComplete;
+              onComplete:(nullable void (^)(NSError *_Nullable error))onComplete;
 
 /** Retrieves the stored data for the given key.
  *
@@ -65,6 +88,34 @@ NS_ASSUME_NONNULL_BEGIN
 - (void)removeLibraryDataForKey:(NSString *)key
                      onComplete:(void (^)(NSError *_Nullable error))onComplete;
 
+/** Returns YES if some events have been stored for the given target, NO otherwise.
+ *
+ * @return YES if the storage contains events for the given target, NO otherwise.
+ */
+- (BOOL)hasEventsForTarget:(GDTCORTarget)target;
+
+/** Returns an iterator object that can be used to iterate over events that match the eventSelector.
+ *
+ * @param eventSelector The event selector to match events with.
+ * @return An iterator instance.
+ */
+- (nullable id<GDTCORStorageEventIterator>)iteratorWithSelector:
+    (GDTCORStorageEventSelector *)eventSelector;
+
+/** Removes events from before the given time.
+ *
+ * @param beforeSnapshot The time at which all events prior should be deleted.
+ * @param onComplete The callback that will be invoked when deleting events is complete.
+ */
+- (void)purgeEventsFromBefore:(GDTCORClock *)beforeSnapshot
+                   onComplete:(void (^)(NSError *_Nullable error))onComplete;
+
+/** Calculates and returns the total disk size that this storage consumes.
+ *
+ * @param onComplete The callback that will be invoked once storage size calculation is complete.
+ */
+- (void)storageSizeWithCallback:(void (^)(uint64_t storageSize))onComplete;
+
 @end
 
 /** Retrieves the storage instance for the given target.

GoogleDataTransport/GDTCORTests/Common/Categories/GDTCORFlatFileStorage+Testing.m

@@ -31,6 +31,12 @@ - (void)reset {
     [[NSFileManager defaultManager] removeItemAtPath:[GDTCORFlatFileStorage libraryDataStoragePath]
                                                error:nil];
   });
+  dispatch_semaphore_t sema = dispatch_semaphore_create(0);
+  [[GDTCORFlatFileStorage sharedInstance] storageSizeWithCallback:^(uint64_t storageSize) {
+    NSAssert(storageSize == 0, @"Storage should contain nothing after a reset");
+    dispatch_semaphore_signal(sema);
+  }];
+  dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
 }
 
 @end

GoogleDataTransport/GDTCORTests/Common/Fakes/GDTCORStorageFake.m

@@ -38,7 +38,7 @@ - (void)libraryDataForKey:(nonnull NSString *)key
 
 - (void)storeLibraryData:(NSData *)data
                   forKey:(nonnull NSString *)key
-              onComplete:(nonnull void (^)(NSError *_Nullable error))onComplete {
+              onComplete:(nullable void (^)(NSError *_Nullable error))onComplete {
   if (onComplete) {
     onComplete(nil);
   }
@@ -51,4 +51,20 @@ - (void)removeLibraryDataForKey:(nonnull NSString *)key
   }
 }
 
+- (BOOL)hasEventsForTarget:(GDTCORTarget)target {
+  return NO;
+}
+
+- (nullable id<GDTCORStorageEventIterator>)iteratorWithSelector:
+    (nonnull GDTCORStorageEventSelector *)eventSelector {
+  return nil;
+}
+
+- (void)purgeEventsFromBefore:(GDTCORClock *)beforeSnapshot
+                   onComplete:(void (^)(NSError *_Nullable error))onComplete {
+}
+
+- (void)storageSizeWithCallback:(void (^)(uint64_t storageSize))onComplete {
+}
+
 @end

GoogleDataTransport/GDTCORTests/Unit/GDTCORFlatFileStorageTest.m

@@ -575,6 +575,53 @@ - (void)testSearchPathsWithEventSelector {
   // TODO(mikehaney24): Implement when events are actually being stored.
 }
 
+/** Tests hasEventsForTarget: returns YES when events are stored and NO otherwise. */
+- (void)testHasEventsForTarget {
+  XCTAssertFalse([[GDTCORFlatFileStorage sharedInstance] hasEventsForTarget:kGDTCORTargetTest]);
+
+  // TODO(mikehaney24): Store events and test for YES.
+}
+
+/** Tests -iteratorWithSelector produces a usable iterator. */
+- (void)testIteratorWithSelector {
+  GDTCORStorageEventSelector *eventSelector =
+      [[GDTCORStorageEventSelector alloc] initWithTarget:kGDTCORTargetTest
+                                          eventIDEqualTo:nil
+                                        mappingIDEqualTo:nil
+                                                qosTiers:nil];
+  id<GDTCORStorageEventIterator> iter =
+      [[GDTCORFlatFileStorage sharedInstance] iteratorWithSelector:eventSelector];
+  XCTAssertNil([iter nextEvent]);
+
+  // TODO(mikehaney24): Store events and test.
+}
+
+/** Tests -purgeEventsFromBefore:onComplete: removes prior to an arbitrary time. */
+- (void)testPurgeEvents {
+  // TODO(mikehaney24): Complete when purge events is implemented.
+}
+
+/** Tests that the size of the storage is returned accurately. */
+- (void)testStorageSizeWithCallback {
+  XCTestExpectation *expectation = [self expectationWithDescription:@"storageSize complete"];
+  [[GDTCORFlatFileStorage sharedInstance] storageSizeWithCallback:^(uint64_t storageSize) {
+    XCTAssertEqual(storageSize, 0);
+    [expectation fulfill];
+  }];
+  [self waitForExpectations:@[ expectation ] timeout:10.0];
+
+  expectation = [self expectationWithDescription:@"storageSize complete"];
+  NSData *data = [@"this is a test" dataUsingEncoding:NSUTF8StringEncoding];
+  [[GDTCORFlatFileStorage sharedInstance] storeLibraryData:data forKey:@"testKey" onComplete:nil];
+  [[GDTCORFlatFileStorage sharedInstance] storageSizeWithCallback:^(uint64_t storageSize) {
+    XCTAssertEqual(storageSize, data.length);
+    [expectation fulfill];
+  }];
+  [self waitForExpectations:@[ expectation ] timeout:10.0];
+
+  // TODO(mikehaney24): Complete this test when events are actually being stored.
+}
+
 /** Tests migration from v1 of the storage format to v2. */
 - (void)testMigrationFromOldVersion {
   static NSString *base64EncodedArchive =