Capture the aggregate size of loaded documents

* Calculate the doc source size
* Fix size variable Init error
* Fixed imports

Author: Raj725

.gitignore

@@ -157,4 +157,4 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/

pebblo_langchain/langchain_community/document_loaders/pebblo.py

@@ -45,12 +45,13 @@ def __init__(
         self.docs = []
         loader_name = str(type(self.loader)).split(".")[-1].split("'")[0]
         self.source_type = get_loader_type(loader_name)
-        self.source_size = self.get_source_size(self.source_path)
+        self.source_path_size = self.get_source_size(self.source_path)
+        self.source_aggr_size = 0
         self.loader_details = {
             "loader": loader_name,
             "source_path": self.source_path,
             "source_type": self.source_type,
-            "source_size": self.source_size,
+            "source_path_size": self.source_path_size,
         }
         # generate app
         self.app = self._get_app_details()
@@ -98,14 +99,16 @@ def _send_loader_doc(self, loading_end=False):
         for doc in doc_content:
             doc_source_path = get_full_path(doc.get("metadata", {}).get("source"))
             doc_source_owner = PebbloSafeLoader.get_file_owner_from_path(doc_source_path)
-            doc_source_size = self.get_source_size(doc_source_path)
+            page_content = doc.get("page_content")
+            doc_source_size = self.calculate_content_size(page_content)
+            self.source_aggr_size += doc_source_size
             docs.append(
                 {
-                    "doc": doc.get("page_content"),
+                    "doc": page_content,
                     "source_path": doc_source_path,
                     "last_modified": doc.get("metadata", {}).get("last_modified"),
                     "file_owner": doc_source_owner,
-                    "source_size": doc_source_size,
+                    "source_path_size": doc_source_size,
                 }
             )
         payload = {
@@ -120,6 +123,7 @@ def _send_loader_doc(self, loading_end=False):
         }
         if loading_end is True:
             payload["loading_end"] = "true"
+            payload["loader_details"]["source_aggr_size"] = self.source_aggr_size
         try:
             payload = Doc.model_validate(payload).model_dump(exclude_unset=True)
         except AttributeError:
@@ -154,6 +158,19 @@ def _send_loader_doc(self, loading_end=False):
         if loading_end is True:
             PebbloSafeLoader.set_loader_sent()
 
+    @staticmethod
+    def calculate_content_size(page_content):
+        """
+        Calculate the content size in bytes:
+        - Encode the string to bytes using a specific encoding (e.g., UTF-8)
+        - Get the length of the encoded bytes.
+        """
+
+        # Encode the content to bytes using UTF-8
+        encoded_content = page_content.encode('utf-8')
+        size = len(encoded_content)
+        return size
+
     def _send_discover(self):
         headers = {"Accept": "application/json", "Content-Type": "application/json"}
         try:
@@ -212,6 +229,7 @@ def get_file_owner_from_path(file_path: str) -> str:
         return file_owner_name
 
     def get_source_size(self, source_path: str) -> int:
+        size = None
         if os.path.isfile(source_path):
             size = os.path.getsize(source_path)
         elif os.path.isdir(source_path):