Filter malformed post-processed docs from file picker (Mintplex-Labs#3853)

Filter malformed post-processed files from UI

Author: timothycarambat

server/utils/files/index.js

@@ -61,9 +61,9 @@ async function viewLocalFiles() {
         );
         filenames[cachefilename] = subfile;
       }
-      const results = await Promise.all(filePromises).then((results) =>
-        results.filter((i) => !!i)
-      );
+      const results = await Promise.all(filePromises)
+        .then((results) => results.filter((i) => !!i)) // Remove null results
+        .then((results) => results.filter((i) => hasRequiredMetadata(i))); // Remove invalid file structures
       subdocs.items.push(...results);
 
       // Grab the pinned workspaces and watched documents for this folder's documents
@@ -433,6 +433,31 @@ async function fileToPickerData({
   };
 }
 
+const REQUIRED_FILE_OBJECT_FIELDS = [
+  "name",
+  "type",
+  "url",
+  "title",
+  "docAuthor",
+  "description",
+  "docSource",
+  "chunkSource",
+  "published",
+  "wordCount",
+  "token_count_estimate",
+];
+
+/**
+ * Checks if a given metadata object has all the required fields
+ * @param {{name: string, type: string, url: string, title: string, docAuthor: string, description: string, docSource: string, chunkSource: string, published: string, wordCount: number, token_count_estimate: number}} metadata - The metadata object to check (fileToPickerData)
+ * @returns {boolean} - Returns true if the metadata object has all the required fields, false otherwise
+ */
+function hasRequiredMetadata(metadata = {}) {
+  return REQUIRED_FILE_OBJECT_FIELDS.every((field) =>
+    metadata.hasOwnProperty(field)
+  );
+}
+
 module.exports = {
   findDocumentInDocuments,
   cachedVectorInformation,