add scopus csv support (#123)

* Add new version of bibx

* Add support for scopus CSV

* Be more descriptive with that error

Author: odarbelaeze

functions/main.py

@@ -4,7 +4,7 @@
 from enum import Enum
 from functools import reduce
 from io import StringIO
-from typing import Any, Dict, List
+from typing import Any, Dict, List, cast
 
 import arrow
 import networkx as nx
@@ -20,10 +20,11 @@
 from firebase_functions.options import MemoryOption
 from firebase_functions.scheduler_fn import ScheduledEvent, on_schedule
 from google.cloud.firestore import DocumentReference, DocumentSnapshot
+from google.cloud.storage.client import Blob
 from pydantic import BaseModel, ValidationError
 
 initialize_app()
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.WARNING)
 
 
 ROOT = "root"
@@ -148,7 +149,11 @@ def get_contents(
     """Get the contents for the files in order to create the graph."""
     names = [f"isi-files/{name}" for name in document_data["files"]]
     logger.info("Reading source files", extra={"names": names})
-    blobs = list(filter(None, [storage.bucket().get_blob(name) for name in names]))
+    bucket = storage.bucket()
+    blobs: list[Blob] = cast(
+        list[Blob],
+        list(filter(None, [bucket.get_blob(blob_name=name) for name in names])),  # type: ignore
+    )
 
     size = 0
     output = {}
@@ -158,7 +163,7 @@ def get_contents(
         size += blob.size or 0
         if (size / 1e6) > max_size_megabytes:
             break
-        output[blob.name] = blob.download_as_text()
+        output[blob.name] = blob.download_as_text()  # type: ignore
     return output
 
 

functions/requirements.txt

@@ -1,5 +1,5 @@
 firebase-admin~=6.6.0
 firebase_functions~=0.4.2
-bibx==0.5.0
+bibx==0.6.0
 pydantic~=2.10.6
 arrow~=1.3.0

package-lock.json

@@ -4,7 +4,7 @@ import { cva, type VariantProps } from "class-variance-authority";
 import * as React from "react";
 
 const buttonVariants = cva(
-  "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm font-medium transition-colors ease-in focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-8 [&_svg]:shrink-0",
+  "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm font-medium transition-colors ease-in focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0",
   {
     variants: {
       variant: {

src/components/ui/Button.tsx

@@ -1,13 +1,14 @@
 import useError from "../../hooks/useError";
 import useUpload from "../../hooks/useUpload";
 import { looksLikeIsi } from "../../utils/isi";
-import { looksLikeScopus } from "../../utils/scopus";
+import { looksLikeScopusCsv } from "../../utils/scopusCsv";
+import { looksLikeScopusRis } from "../../utils/scopusRis";
 import { FC, useCallback } from "react";
 import { useDropzone } from "react-dropzone";
 import { useLocation, useNavigate } from "react-router-dom";
 
 const FileErrorMap = {
-  not_supported: "does not look like a valid ISI file",
+  not_supported: "does not look like a valid CSV, BIB, or ISI file",
   max_size: (maxSize: number) => `is too big to process (max. ${maxSize}MB)`,
 };
 
@@ -35,7 +36,11 @@ const FileDropper: FC<FileDropperProps> = ({ maxSize }) => {
           .map((file) => file.text().then((text) => ({ text, file }))),
       ).then((data) => {
         data.forEach(({ text, file }) => {
-          if (looksLikeIsi(text) || looksLikeScopus(text)) {
+          if (
+            looksLikeIsi(text) ||
+            looksLikeScopusRis(text) ||
+            looksLikeScopusCsv(text)
+          ) {
             upload(Object(file).name, file);
           } else {
             error(Object(file).name, file, FileErrorMap.not_supported);

src/components/upload/FileDropper.tsx

@@ -1,7 +1,8 @@
 import { FileMetadata } from "../types/fileMetadata";
 import { mostCommon } from "./arrays";
 import * as isi from "./isi";
-import * as scopus from "./scopus";
+import * as scopusCsv from "./scopusCsv";
+import * as scopusRis from "./scopusRis";
 import md5 from "md5";
 
 const metadata = async (name: string, blob: Blob): Promise<FileMetadata> => {
@@ -20,14 +21,26 @@ const metadata = async (name: string, blob: Blob): Promise<FileMetadata> => {
     };
   }
 
-  if (scopus.looksLikeScopus(content)) {
+  if (scopusRis.looksLikeScopusRis(content)) {
     return {
       name,
       blob,
       hash,
-      keywords: mostCommon(scopus.keywords(content), 3),
-      articles: scopus.countArticles(content),
-      citations: scopus.countReferences(content),
+      keywords: mostCommon(scopusRis.keywords(content), 3),
+      articles: scopusRis.countArticles(content),
+      citations: scopusRis.countReferences(content),
+      valid: true,
+    };
+  }
+
+  if (scopusCsv.looksLikeScopusCsv(content)) {
+    return {
+      name,
+      blob,
+      hash,
+      keywords: mostCommon(scopusCsv.keywords(content), 3),
+      articles: scopusCsv.countArticles(content),
+      citations: scopusCsv.countReferences(content),
       valid: true,
     };
   }

src/utils/metadata.ts

@@ -0,0 +1,49 @@
+import Papa from "papaparse";
+import { z } from "zod";
+
+const fileSchema = z.array(
+  z.object({
+    Authors: z.string(),
+    "Author Keywords": z.string(),
+    "Index Keywords": z.string(),
+    References: z.string(),
+  }),
+);
+
+type CsvFileType = z.infer<typeof fileSchema>;
+
+const readCsvText = (text: string): CsvFileType => {
+  const { data } = Papa.parse(text, { header: true, skipEmptyLines: true });
+  return fileSchema.parse(data);
+};
+
+const looksLikeScopusCsv = (text: string): boolean => {
+  try {
+    const data = readCsvText(text);
+    return data.length > 0;
+  } catch {
+    return false;
+  }
+};
+
+const keywords = (text: string): string[] => {
+  const data = readCsvText(text);
+  const keywords = data.flatMap((item) => [
+    ...item["Author Keywords"].split(";"),
+    ...item["Index Keywords"].split(";"),
+  ]);
+  return Array.from(new Set(keywords));
+};
+
+const countArticles = (text: string): number => {
+  const data = readCsvText(text);
+  return data.length;
+};
+
+const countReferences = (text: string): number => {
+  const data = readCsvText(text);
+  const references = data.flatMap((item) => item["References"].split("; "));
+  return references.length;
+};
+
+export { looksLikeScopusCsv, keywords, countArticles, countReferences };

src/utils/scopusCsv.ts

@@ -1,6 +1,6 @@
 const RIS_PATTERN = /^(?<key>[A-Z0-9]{2})[ ]{2}-[ ](?<value>.*)$/;
 
-const looksLikeScopus = (content: string): boolean => {
+const looksLikeScopusRis = (content: string): boolean => {
   let currentKey = null;
   for (const line of content.split("\n")) {
     if (!line) {
@@ -75,4 +75,4 @@ const countReferences = (content: string): number =>
       { counting: false, count: 0 },
     ).count;
 
-export { looksLikeScopus, keywords, countArticles, countReferences };
+export { looksLikeScopusRis, keywords, countArticles, countReferences };