Saad Stat Tracking #1

Author: SaadArdati

example/pubspec.yaml

@@ -16,6 +16,10 @@ dependencies:
   codelessly_sdk:
     path: ../
 
+dependency_overrides:
+  archive: ^3.6.1
+  win32: ^5.5.3
+
 dev_dependencies:
   flutter_test:
     sdk: flutter

lib/codelessly_sdk.dart

@@ -17,6 +17,7 @@ export 'src/data/web_data_repository.dart';
 export 'src/functions/functions.dart';
 export 'src/logging/codelessly_logger.dart';
 export 'src/logging/error_handler.dart';
+export 'src/logging/stat_tracker.dart';
 export 'src/model/asset_model.dart';
 export 'src/model/auth_data.dart';
 export 'src/model/custom_component.dart';

lib/src/codelessly.dart

@@ -128,6 +128,11 @@ class Codelessly {
   final StreamController<CStatus> _statusStreamController =
       StreamController.broadcast()..add(CStatus.empty());
 
+  final StatTracker _tracker = FirestoreStatTracker();
+
+  /// Tracks statistics of various operations in the SDK.
+  StatTracker get tracker => _tracker;
+
   /// Returns a stream of status updates for this SDK instance.
   Stream<CStatus> get statusStream => _statusStreamController.stream;
 
@@ -557,11 +562,16 @@ class Codelessly {
             cacheManager: _cacheManager!,
             authManager: _authManager!,
             networkDataRepository: FirebaseDataRepository(
-                firestore: firebaseFirestore, config: _config!),
-            localDataRepository:
-                LocalDataRepository(cacheManager: _cacheManager!),
+              firestore: firebaseFirestore,
+              tracker: tracker,
+              config: _config!,
+            ),
+            localDataRepository: LocalDataRepository(
+              cacheManager: _cacheManager!,
+            ),
             firebaseFirestore: firebaseFirestore,
             errorHandler: errorHandler,
+            tracker: tracker,
           );
 
       // Create the preview data manager.
@@ -572,11 +582,16 @@ class Codelessly {
             cacheManager: _cacheManager!,
             authManager: _authManager!,
             networkDataRepository: FirebaseDataRepository(
-                firestore: firebaseFirestore, config: _config!),
-            localDataRepository:
-                LocalDataRepository(cacheManager: _cacheManager!),
+              firestore: firebaseFirestore,
+              config: _config!,
+              tracker: tracker,
+            ),
+            localDataRepository: LocalDataRepository(
+              cacheManager: _cacheManager!,
+            ),
             firebaseFirestore: firebaseFirestore,
             errorHandler: errorHandler,
+            tracker: tracker,
           );
 
       // Create the template data manager. This is always automatically
@@ -591,11 +606,16 @@ class Codelessly {
             cacheManager: _cacheManager!,
             authManager: _authManager!,
             networkDataRepository: FirebaseDataRepository(
-                firestore: firebaseFirestore, config: _config!),
-            localDataRepository:
-                LocalDataRepository(cacheManager: _cacheManager!),
+              firestore: firebaseFirestore,
+              config: _config!,
+              tracker: tracker,
+            ),
+            localDataRepository: LocalDataRepository(
+              cacheManager: _cacheManager!,
+            ),
             firebaseFirestore: firebaseFirestore,
             errorHandler: errorHandler,
+            tracker: tracker,
           );
 
       _updateStatus(CStatus.loading(CLoadingState.createdManagers));
@@ -620,6 +640,10 @@ class Codelessly {
         await _authManager!.init();
         _config!.publishSource = _authManager!.getBestPublishSource(_config!);
 
+        if (_authManager?.authData?.projectId case String projectId) {
+          tracker.init(projectId);
+        }
+
         _updateStatus(CStatus.loading(CLoadingState.initializedAuth));
       } else {
         log('A slug was provided. Acutely skipping authentication.');
@@ -667,6 +691,10 @@ class Codelessly {
         _authManager!.init().then((_) async {
           if (_authManager!.authData == null) return;
 
+          if (_authManager?.authData?.projectId case String projectId) {
+            tracker.init(projectId);
+          }
+
           log('[POST-INIT] Background authentication succeeded. Initializing layout storage.');
           await dataManager
               .onPublishModelLoaded(_authManager!.authData!.projectId);

lib/src/constants.dart

@@ -53,6 +53,16 @@ const String publishPath = 'publish';
 const String previewPath = 'publish_preview';
 const String templatesPath = 'templates';
 
+/// STAT TRACKING
+const String statsCollection = 'stats';
+const String lostStatsDoc = '_lost';
+const String writesField = 'writes';
+const String readsField = 'reads';
+const String bundleDownloadsField = 'bundle_downloads';
+const String fontDownloadsField = 'font_downloads';
+const String actionsField = 'actions';
+const String cloudActionsField = 'cloud_actions';
+
 /// The template url for the svg icons.
 /// {{style}}: The style of the icon. e.g. materialiconsoutlined
 /// {{name}}: The name of the icon. e.g. home

lib/src/data/cloud_database.dart

@@ -2,7 +2,7 @@ import 'dart:async';
 
 import 'package:cloud_firestore/cloud_firestore.dart';
 import 'package:codelessly_api/codelessly_api.dart';
-import 'package:flutter/cupertino.dart';
+import 'package:flutter/widgets.dart';
 
 import '../../codelessly_sdk.dart';
 import '../utils/constants.dart';
@@ -41,6 +41,8 @@ abstract class CloudDatabase extends ChangeNotifier {
   /// This cannot be an empty string.
   final String identifier;
 
+  /// The source of the publish, this is used to determine the root collection
+  /// to use for the cloud database.
   final PublishSource publishSource;
 
   /// Creates a new instance of with the given [identifier].
@@ -195,12 +197,20 @@ class FirestoreCloudDatabase extends CloudDatabase {
   /// Reference to the Firestore instance.
   final FirebaseFirestore firestore;
 
+  /// Reference to the StatTracker instance, used to track reads and writes.
+  final StatTracker tracker;
+
   /// Subscriptions to the streams that are being listened to.
   final List<StreamSubscription> _subscriptions = [];
 
   /// Creates a [FirestoreCloudDatabase] with the given [identifier] and
   /// [firestore] instance.
-  FirestoreCloudDatabase(super.identifier, this.firestore, super.publishSource);
+  FirestoreCloudDatabase(
+    super.identifier,
+    super.publishSource, {
+    required this.firestore,
+    required this.tracker,
+  });
 
   void log(String message) => logger.log(_label, message);
 
@@ -213,12 +223,14 @@ class FirestoreCloudDatabase extends CloudDatabase {
 
     // Create project doc if missing.
     final snapshot = await rootRef.get();
+    tracker.trackRead();
 
     // Do nothing if project doc exists.
     if (snapshot.exists) return;
 
     // Create project doc if it does not exist.
     await rootRef.set({'project': identifier});
+    tracker.trackWrite();
 
     logger.log(_label, 'Done initializing for $identifier');
   }
@@ -295,6 +307,8 @@ class FirestoreCloudDatabase extends CloudDatabase {
     if (autoGenerateId) {
       // if autoGenerateId is true, then skipCreationIfDocumentExists and docId is ignored.
       final document = await rootRef.collection(path).add(value);
+      tracker.trackWrite();
+
       logger.log(_label, 'Document added: ${document.path}');
 
       // Sanitize data afterwards because we didn't have the docId before.
@@ -313,6 +327,8 @@ class FirestoreCloudDatabase extends CloudDatabase {
 
     // Get snapshot to check if document exists.
     final snapshot = await docRef.get();
+    tracker.trackRead();
+
     if (skipCreationIfDocumentExists && snapshot.exists) {
       // if skipCreationIfDocumentExists is true, check if document exists.
       // if document exists, then return.
@@ -322,6 +338,8 @@ class FirestoreCloudDatabase extends CloudDatabase {
 
     // Set document.
     await docRef.set(value);
+    tracker.trackWrite();
+
     logger.log(_label, 'Document added: ${docRef.path}/$documentId');
     return true;
   }
@@ -344,36 +362,50 @@ class FirestoreCloudDatabase extends CloudDatabase {
 
     // TODO: Should we do update instead of set?
     await docRef.set(value, SetOptions(merge: true));
+    tracker.trackWrite();
+
     logger.log(_label, 'Document updated: ${docRef.path}');
     return true;
   }
 
   @override
   Future<bool> removeDocument(String path, String documentId) async {
     final docRef = getDocPath(path, documentId);
+
     final snapshot = await docRef.get();
+    tracker.trackRead();
+
     // TODO: Do we have to check for existence?
     if (!snapshot.exists) return false;
+
     await docRef.delete();
+    tracker.trackWrite();
+
     return true;
   }
 
   @override
   Future<Map<String, dynamic>> getDocumentData(
       String path, String documentId) async {
     final docRef = getDocPath(path, documentId);
+
     final snapshot = await docRef.get();
+    tracker.trackRead();
+
     final data = snapshot.data() ?? {};
     return sanitizeCloudDataForUse(data, docId: snapshot.id);
   }
 
   @override
   Stream<Map<String, dynamic>> streamDocument(String path, String documentId) {
     final docRef = getDocPath(path, documentId);
-    return docRef.snapshots().map((snapshot) =>
-        snapshot.data()?.let(
-            (value) => sanitizeCloudDataForUse(value, docId: snapshot.id)) ??
-        {});
+    return docRef.snapshots().map((snapshot) {
+      tracker.trackRead();
+
+      return snapshot.data()?.let(
+              (value) => sanitizeCloudDataForUse(value, docId: snapshot.id)) ??
+          {};
+    });
   }
 
   @override
@@ -391,6 +423,8 @@ class FirestoreCloudDatabase extends CloudDatabase {
     // Listen to the stream and update the variable.
     final subscription = stream.listen(
       (data) {
+        tracker.trackRead();
+
         logger.log(_label,
             'Document stream update from cloud storage: $path/$documentId');
         logger.log(_label,
@@ -466,6 +500,8 @@ class FirestoreCloudDatabase extends CloudDatabase {
     // Listen to the stream and update the variable.
     final subscription = stream.listen(
       (snapshot) {
+        tracker.trackRead();
+
         final docs = snapshot.docs
             .map((doc) => sanitizeCloudDataForUse(doc.data(), docId: doc.id))
             .toList();

lib/src/data/data_manager.dart

@@ -46,16 +46,32 @@ class DataManager {
   /// The auth manager to use. By default, it is [CodelesslyAuthManager].
   final AuthManager authManager;
 
+  // TODO(Saad): This is only used to initialize CloudDatabase. We may be able
+  //  to decouple it.
   /// The firestore instance to use.
   final FirebaseFirestore firebaseFirestore;
 
   /// The error handler to use.
   final CodelesslyErrorHandler errorHandler;
 
+  // TODO(Saad): This is only used to initialize CloudDatabase. We may be able
+  //  to decouple it.
+  /// The stat tracker to use, used to track various reads and writes in this
+  /// data manager.
+  final StatTracker tracker;
+
   SDKPublishModel? _publishModel;
 
+  // TODO(Saad): LocalDatabase is only initialized and never truly accessed by
+  //  the DataManager itself. We can decouple it by moving it to the
+  //  [Codelessly] level. Maybe through a new shared manager for it and
+  //  CloudDatabase.
   LocalDatabase? _localDatabase;
 
+  // TODO(Saad): CloudDatabase is only initialized and never truly accessed by
+  //  the DataManager itself. We can decouple it by moving it to the
+  //  [Codelessly] level. Maybe through a new shared manager for it and
+  //  LocalDatabase.
   CloudDatabase? _cloudDatabase;
 
   /// The local storage instance used by this data manager.
@@ -102,6 +118,7 @@ class DataManager {
     required this.localDataRepository,
     required this.firebaseFirestore,
     required this.errorHandler,
+    required this.tracker,
     SDKPublishModel? publishModel,
   }) : _publishModel = publishModel;
 
@@ -479,12 +496,17 @@ class DataManager {
     return HiveLocalDatabase(box, identifier: projectId);
   }
 
+  // TODO(Saad): We can de-couple [firebaseFirestore] and [tracker] from
+  //  [DataManager] by delegating this initialization to the creator of
+  //  [DataManager], maybe through a callback so that DataManager can decide
+  //  exactly when to initialize this.
   Future<CloudDatabase> initializeCloudStorage(
       {required String projectId}) async {
     final instance = FirestoreCloudDatabase(
       projectId,
-      firebaseFirestore,
       config.publishSource,
+      firestore: firebaseFirestore,
+      tracker: tracker,
     );
     // initialize cloud storage.
     await instance.init();