feat(documents): improve document handling

Signed-off-by: Kipruto <43873157+kelvinkipruto@users.noreply.github.com>

Author: kelvinkipruto

src/collections/Documents.ts

@@ -19,6 +19,13 @@ export const Documents: CollectionConfig = {
       name: 'url',
       type: 'text',
     },
+    {
+      name: 'docURL', // only used to download file from airtable
+      type: 'text',
+      admin: {
+        hidden: true,
+      },
+    },
     {
       name: 'file',
       type: 'upload',
@@ -115,13 +122,19 @@ export const Documents: CollectionConfig = {
     },
     {
       type: 'tabs',
+      admin: {
+        readOnly: true,
+      },
       tabs: [
         {
           label: 'Extracted Text',
           fields: [
             {
               name: 'extractedText',
               type: 'richText',
+              admin: {
+                readOnly: true,
+              },
             },
           ],
         },
@@ -153,31 +166,25 @@ export const Documents: CollectionConfig = {
                 },
                 {
                   type: 'row',
-                  admin: {
-                    readOnly: true,
-                  },
                   fields: [
                     {
                       name: 'uniqueId',
                       type: 'text',
-                      // admin: {
-                      //   readOnly: true,
-                      // },
+                      admin: {
+                        hidden: true,
+                      },
                     },
                     {
                       name: 'checkMediaId',
                       type: 'text',
-                      // admin: {
-                      //   readOnly: true,
-                      // },
+                      admin: {
+                        hidden: true,
+                      },
                     },
                     {
                       name: 'checkMediaURL',
                       label: 'CheckMedia URL',
                       type: 'text',
-                      // admin: {
-                      //   readOnly: true,
-                      // },
                     },
                   ],
                 },

src/payload-types.ts

@@ -186,6 +186,7 @@ export interface Document {
   id: string;
   title: string;
   url?: string | null;
+  docURL?: string | null;
   file?: (string | null) | Media;
   politicalEntity?: string | null;
   country?: string | null;
@@ -468,6 +469,7 @@ export interface MediaSelect<T extends boolean = true> {
 export interface DocumentsSelect<T extends boolean = true> {
   title?: T;
   url?: T;
+  docURL?: T;
   file?: T;
   politicalEntity?: T;
   country?: T;

src/tasks/aiSummarizer.ts

@@ -57,6 +57,7 @@ export const AISummarizer: TaskConfig<'aiSummarizer'> = {
     - Source field must contain exact quotes from the document text
     - Each promise should have clear supporting evidence in the source quotes
     - Group similar promises under appropriate categories
+    - Reply in the language the document is written in
 
     **Document:**
     

src/tasks/downloadDocuments.ts

@@ -64,6 +64,7 @@ export const DownloadDocuments: TaskConfig<'downloadDocuments'> = {
         },
         select: {
           url: true,
+          docURL: true,
         },
       })
 
@@ -72,14 +73,17 @@ export const DownloadDocuments: TaskConfig<'downloadDocuments'> = {
       for (const doc of documents) {
         try {
           logger.debug('Processing document', { id: doc.id })
-          const { url } = doc
+          const { url, docURL } = doc
 
-          if (!url) {
+          // priritize file uploaded to airtable
+          const urlToFetch = docURL || url
+
+          if (!urlToFetch) {
             logger.warn('Document has no URL', { id: doc.id })
             continue
           }
 
-          const res = await fetch(url)
+          const res = await fetch(urlToFetch)
 
           if (!res.ok) {
             throw new Error(`Failed to fetch document: ${res.status} ${res.statusText}`)

src/tasks/fetchAirtableDocuments.ts

@@ -68,7 +68,8 @@ export const FetchAirtableDocuments: TaskConfig<'fetchAirtableDocuments'> = {
             yearFrom: doc.yearFrom,
             yearTo: doc.yearTo,
             airtableID: doc.id,
-            url: doc.document.length ? doc.document[0] : doc.uRL,
+            url: doc.uRL,
+            docURL: doc.document[0],
           },
         })
       }),

src/tasks/uploadToMeedan.ts

@@ -1,7 +1,7 @@
 import { TaskConfig } from 'payload'
 import { createFactCheckClaim } from '@/lib/meedan'
 import { markDocumentAsProcessed } from '@/lib/airtable'
-import { Document } from '@/payload-types'
+import { Document, Media } from '@/payload-types'
 
 export const UploadToMeedan: TaskConfig<'uploadToMeedan'> = {
   slug: 'uploadToMeedan',
@@ -38,7 +38,7 @@ export const UploadToMeedan: TaskConfig<'uploadToMeedan'> = {
           },
         },
         limit: -1,
-        depth: 0,
+        depth: 2,
       })
 
       logger.info(`Found ${documents.length} documents to process`)
@@ -106,6 +106,8 @@ export const UploadToMeedan: TaskConfig<'uploadToMeedan'> = {
               ${extraction.source}
               `.trim()
 
+              const downloadedFile = doc.file as Media
+
               const response = await createFactCheckClaim({
                 apiKey: meedanAPIKey,
                 teamId,
@@ -114,7 +116,7 @@ export const UploadToMeedan: TaskConfig<'uploadToMeedan'> = {
                 claimDescription: extraction.summary,
                 factCheck: {
                   title: extraction.summary,
-                  url: doc.url || '',
+                  url: doc.url || downloadedFile.url || doc.docURL || '',
                   language: doc.language || '',
                   publish_report: false,
                 },