updated chunking

Author: Gautam-Rajeev

src/chunking/MPNet/local/api.py

@@ -4,20 +4,18 @@
 import aiohttp
 import pandas as pd
 import io
-from PyPDF2 import PdfReader
+import fitz
 import os 
 
 def extract_text_from_txt(txt_path):
     with open(txt_path, 'r', encoding='utf-8') as file:
         return file.read()
 
 def extract_text_from_pdf(pdf_path):
-    reader = PdfReader(pdf_path)
-    number_of_pages = len(reader.pages)
+    doc = fitz.open(pdf_path) # open a document
     all_text = ""
-
-    for page in reader.pages:
-        all_text += page.extract_text()
+    for page in doc: # iterate the document pages
+        all_text += page.get_text("text")
 
     return all_text
 
@@ -49,8 +47,8 @@ async def embed():
                 text_data = uploaded_file.stream.read().decode('utf-8')
             elif file_extension == '.pdf':
                 pdf_file_stream = io.BytesIO(uploaded_file.stream.read())
-                reader = PdfReader(pdf_file_stream)
-                pages = [(i, page.extract_text()) for i, page in enumerate(reader.pages)]  # Modified line
+                doc = fitz.open("pdf", pdf_file_stream.getvalue())
+                pages = [(i, page.get_text("text")) for i, page in enumerate(doc)]  # Modified line
                 text_data = pages
             else:
                 return (print('Wrong format of file submitted'))

src/chunking/MPNet/local/model.py

@@ -8,7 +8,7 @@
 from sklearn.metrics.pairwise import cosine_similarity
 import math
 from scipy.signal import argrelextrema
-from PyPDF2 import PdfReader
+import fitz
 from request import ModelRequest
 import torch
 import nltk 
@@ -127,12 +127,10 @@ def activate_similarities(self,similarities:np.array, p_size=10)->np.array:
 
 
 def extract_text_from_pdf(pdf_path):
-    reader = PdfReader(pdf_path)
-    number_of_pages = len(reader.pages)
+    doc = fitz.open(pdf_path) # open a document
     all_text = ""
-
-    for page in reader.pages:
-        all_text += page.extract_text()
+    for page in doc: # iterate the document pages
+        all_text += page.get_text("text")
 
     return all_text
 

src/chunking/MPNet/local/requirements.txt

@@ -7,4 +7,4 @@ segeval
 numpy
 nltk
 scipy
-PyPDF2
+PyMuPDF