Add updated notebook and data

Author: justin-lee-cohere

notebooks/Document_Parsing_For_Enterprises.ipynb

@@ -60,9 +60,9 @@
     "\n",
     "Before we dive into the technical weeds, we need to set up the notebook's runtime and filesystem environments. The code cells below do the following:\n",
     "- Install required libraries\n",
-    "- Fetch the PDF dataset that we will be working with `fda-approved-drug.pdf`\n",
-    "  - External link: https://www.accessdata.fda.gov/drugsatfda_docs/label/2023/215500s000lbl.pdf\n",
-    "- Fetch pre-existing versions of the parsed documents for each solution. For the sake of the notebook, this allows readers to see the final results without having to set up the required infrastructure\n",
+    "- Confirm that data dependencies from the GitHub repo have been downloaded. These will be under `data/document-parsing` and contain the following:\n",
+    "    - the PDF document that we will be working with, `fda-approved-drug.pdf` (this can also be found here: https://www.accessdata.fda.gov/drugsatfda_docs/label/2023/215500s000lbl.pdf)\n",
+    "    - precomputed parsed documents for each parsing solution. While the point of this notebook is to illustrate how this is done, we provide the parsed final results to allow readers to skip ahead to the RAG section without having to set up the required infrastructure for each solution.)\n",
     "- Add utility functions needed for later sections"
    ]
   },
@@ -76,7 +76,7 @@
    "source": [
     "%%capture\n",
     "! sudo apt install tesseract-ocr poppler-utils\n",
-    "! pip install cohere hnswlib google-cloud-documentai google-cloud-storage boto3 langchain-text-splitters llama_parse pytesseract pdf2image pandas\n"
+    "! pip install cohere fsspec hnswlib google-cloud-documentai google-cloud-storage boto3 langchain-text-splitters llama_parse pytesseract pdf2image pandas\n"
    ]
   },
   {
@@ -87,36 +87,28 @@
    },
    "outputs": [],
    "source": [
-    "\"\"\"\n",
-    "Create filesystem\n",
-    "\"\"\"\n",
-    "\n",
-    "from pathlib import Path\n",
-    "\n",
-    "data_dir = \"data\"\n",
+    "data_dir = \"data/document-parsing\"\n",
     "source_filename = \"example-drug-label\"\n",
-    "extension = \"pdf\"\n",
-    "Path(data_dir).mkdir(parents=True, exist_ok=True)"
+    "extension = \"pdf\""
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "id": "3Em5H09-Yp19"
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
-    "\"\"\"\n",
-    "Fetch notebook's data\n",
-    "\"\"\"\n",
-    "\n",
-    "import fsspec\n",
     "from pathlib import Path\n",
     "\n",
-    "destination = Path(data_dir)\n",
-    "fs = fsspec.filesystem(\"github\", org=\"gchatz22\", repo=\"temp-cohere-resources\")\n",
-    "fs.get(fs.ls(data_dir), destination.as_posix(), recursive=True)"
+    "sources = [\"gcp\", \"aws\", \"unstructured-io\", \"llamaparse-text\", \"llamaparse-markdown\", \"pytesseract\"]\n",
+    "\n",
+    "filenames = [\"{}-parsed-fda-approved-drug.txt\".format(source) for source in sources]\n",
+    "filenames.append(\"fda-approved-drug.pdf\")\n",
+    "\n",
+    "for filename in filenames:   \n",
+    "    file_path = Path(f\"{data_dir}/{filename}\")\n",
+    "    if file_path.is_file() == False:\n",
+    "        print(f\"File {filename} not found at {data_dir}!\")"
    ]
   },
   {
@@ -138,9 +130,8 @@
    "outputs": [],
    "source": [
     "def store_document(path: str, doc_content: str):\n",
-    "  file = open(path, 'w')\n",
-    "  file.write(doc_content)\n",
-    "  file.close()"
+    "    with open(path, 'w') as f:\n",
+    "      f.write(doc_content)"
    ]
   },
   {
@@ -229,7 +220,7 @@
     "id": "cxwJ_jZpgNDo"
    },
    "source": [
-    "e#### Parsing the document\n",
+    "#### Parsing the document\n",
     "\n",
     "The following block can be executed in one of two ways\n",
     "1. Inside a Google Vertex AI environment\n",
@@ -432,9 +423,8 @@
    "outputs": [],
    "source": [
     "filename = \"gcp-parsed-{}.txt\".format(source_filename)\n",
-    "doc = open(\"{}/{}\".format(data_dir, filename), \"r\")\n",
-    "parsed_document = doc.read()\n",
-    "doc.close()\n",
+    "with open(\"{}/{}\".format(data_dir, filename), \"r\") as doc:\n",
+    "    parsed_document = doc.read()\n",
     "\n",
     "print(parsed_document[:1000])"
    ]
@@ -476,7 +466,7 @@
     "At minimum, you will need access to the following AWS resources to get started:\n",
     "\n",
     "- Textract\n",
-    "- an S3 bucket containing the document(s) to process - in this case, our `Example Label.pdf` file\n",
+    "- an S3 bucket containing the document(s) to process - in this case, our `example-drug-label.pdf` file\n",
     "- an SNS topic that Textract can publish to. This is used to send a notification that parsing is complete.\n",
     "- an IAM role that Textract will assume, granting access to the S3 bucket and SNS topic\n",
     "\n",
@@ -825,7 +815,7 @@
     "sns_topic_arn = \"your-sns-arn\" # this can be found under the topic you created in the Amazon SNS dashboard\n",
     "sns_role_arn = \"sns-role-arn\" # this is an IAM role that allows Textract to interact with SNS\n",
     "\n",
-    "file_name = \"Example Label.pdf\""
+    "file_name = \"example-drug-label.pdf\""
    ]
   },
   {
@@ -979,9 +969,8 @@
    "outputs": [],
    "source": [
     "filename = \"aws-parsed-{}.txt\".format(source_filename)\n",
-    "doc = open(\"{}/{}\".format(data_dir, filename), \"r\")\n",
-    "parsed_document = doc.read()\n",
-    "doc.close()\n",
+    "with open(\"{}/{}\".format(data_dir, filename), \"r\") as doc:\n",
+    "    parsed_document = doc.read()\n",
     "\n",
     "print(parsed_document[:1000])"
    ]
@@ -1031,18 +1020,19 @@
     "parsed_documents = []\n",
     "\n",
     "input_path = \"{}/{}.{}\".format(data_dir, source_filename, extension)\n",
-    "file_data = open(input_path, 'rb')\n",
-    "response = requests.post(\n",
-    "    url=UNSTRUCTURED_URL,\n",
-    "    files={\"files\": (\"{}.{}\".format(source_filename, extension), file_data)},\n",
-    "    data={\n",
-    "        \"output_format\": (None, \"application/json\"),\n",
-    "        \"stratergy\": \"hi_res\",\n",
-    "        \"pdf_infer_table_structure\": \"true\",\n",
-    "        \"include_page_breaks\": \"true\"\n",
-    "    },\n",
-    "    headers={\"Accept\": \"application/json\"}\n",
-    ")\n",
+    "with open(input_path, 'rb') as file_data:\n",
+    "    response = requests.post(\n",
+    "        url=UNSTRUCTURED_URL,\n",
+    "        files={\"files\": (\"{}.{}\".format(source_filename, extension), file_data)},\n",
+    "        data={\n",
+    "            \"output_format\": (None, \"application/json\"),\n",
+    "            \"stratergy\": \"hi_res\",\n",
+    "            \"pdf_infer_table_structure\": \"true\",\n",
+    "            \"include_page_breaks\": \"true\"\n",
+    "        },\n",
+    "        headers={\"Accept\": \"application/json\"}\n",
+    "    )\n",
+    "\n",
     "parsed_response = response.json()\n",
     "\n",
     "parsed_document = \" \".join([parsed_entry[\"text\"] for parsed_entry in parsed_response])\n",
@@ -1083,9 +1073,8 @@
    "outputs": [],
    "source": [
     "filename = \"unstructured-io-parsed-{}.txt\".format(source_filename)\n",
-    "doc = open(\"{}/{}\".format(data_dir, filename), \"r\")\n",
-    "parsed_document = doc.read()\n",
-    "doc.close()\n",
+    "with open(\"{}/{}\".format(data_dir, filename), \"r\") as doc:\n",
+    "    parsed_document = doc.read()\n",
     "\n",
     "print(parsed_document[:1000])"
    ]
@@ -1233,10 +1222,10 @@
     "# Text parsing\n",
     "\n",
     "filename = \"llamaparse-text-parsed-{}.txt\".format(source_filename)\n",
-    "doc = open(\"{}/{}\".format(data_dir, filename), \"r\")\n",
-    "parsed_document = doc.read()\n",
-    "doc.close()\n",
     "\n",
+    "with open(\"{}/{}\".format(data_dir, filename), \"r\") as doc:\n",
+    "    parsed_document = doc.read()\n",
+    "    \n",
     "print(parsed_document[:1000])"
    ]
   },
@@ -1251,10 +1240,9 @@
     "# Markdown parsing\n",
     "\n",
     "filename = \"llamaparse-markdown-parsed-fda-approved-drug.txt\"\n",
-    "doc = open(\"{}/{}\".format(data_dir, filename), \"r\")\n",
-    "parsed_document = doc.read()\n",
-    "doc.close()\n",
-    "\n",
+    "with open(\"{}/{}\".format(data_dir, filename), \"r\") as doc:\n",
+    "    parsed_document = doc.read()\n",
+    "    \n",
     "print(parsed_document[:1000])"
    ]
   },
@@ -1401,9 +1389,8 @@
    "outputs": [],
    "source": [
     "filename = \"pytesseract-parsed-{}.txt\".format(source_filename)\n",
-    "doc = open(\"{}/{}\".format(data_dir, filename), \"r\")\n",
-    "parsed_document = doc.read()\n",
-    "doc.close()\n",
+    "with open(\"{}/{}\".format(data_dir, filename), \"r\") as doc:\n",
+    "    parsed_document = doc.read()\n",
     "\n",
     "print(parsed_document[:1000])"
    ]
@@ -1414,8 +1401,8 @@
     "id": "SCbkT4oZSfs9"
    },
    "source": [
-    "## Document Questions\n",
     "<a name=\"document-questions\"></a>\n",
+    "## Document Questions\n",
     "\n",
     "We can now ask a set of simple + complex questions and see how each parsing solution performs with Command-R. The questions are\n",
     "- **What are the most common adverse reactions of Iwilfin?**\n",
@@ -1493,9 +1480,8 @@
     "\n",
     "documents = []\n",
     "\n",
-    "doc = open(\"{}/{}-parsed-fda-approved-drug.txt\".format(data_dir, source), \"r\")\n",
+    "with open(\"{}/{}-parsed-fda-approved-drug.txt\".format(data_dir, source), \"r\") as doc:\n",
     "doc_content = doc.read()\n",
-    "doc.close()\n",
     "\n",
     "\"\"\"\n",
     "Personal notes on chunking\n",