intel

Author: ANGELOANTU7

Backend/Final_Processor.py

@@ -10,9 +10,9 @@ async def fetch_data(link:str,receiver_email: str ):
     async with httpx.AsyncClient(timeout=3000.0) as client:
         # Make the first API call
         response1 = await client.get(f"{link}/process_files?user={receiver_email}")
-
+        print(response1)
         response2 = await client.get(f"{link}/sorter?user={receiver_email}")
-        
+        print(response2)
         response3 = await client.get(f"{link}/card-json?user={receiver_email}")
 
         response4 = await client.get(f"{link}/note_gen?user={receiver_email}")

Backend/Final_Questionare_Creater.py

@@ -37,7 +37,7 @@ async def summarize_file(bucket_name: str, file_key: str, file_name:str):
         file_name=file_name.split(".txt")[0]
         response = s3.get_object(Bucket=bucket_name, Key=file_key)
         file_content = response['Body'].read().decode('utf-8')
-        prompt = f'create 10 mcq question with 4 option on topic: {file_name} , based on text:{file_content} \n \n output should strictly be a json with array of (question,options,correct option)'
+        prompt = f'create 10 mcq question with 4 option on topic: {file_name} , based on text:{file_content} \n \n output should strictly be a json with array of (question,options,correct option) correct option should be a integer telling which mcq is correct'
         response = openai.ChatCompletion.create(
         model="gpt-3.5-turbo",
         messages=[

Backend/Intel-oneAPI/Final_Notegen.py

@@ -0,0 +1,70 @@
+import os
+import boto3
+from fastapi import APIRouter
+import openai
+from intel_extension_pytorch import PyTorchExtension
+from intel_extension_tensorflow import TensorFlowExtension
+from intel_optimization_xgboost import XGBoostOptimizer
+from intel_optimization_modin import ModinOptimizer
+
+app = APIRouter()
+s3_access_key = "<your_s3_access_key>"
+s3_secret_access_key = "<your_s3_secret_access_key>"
+s3_bucket_name = "learnmateai"
+
+s3 = boto3.client("s3", aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_access_key)
+
+# Set up OpenAI API credentials
+openai.api_key = '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'
+
+# Initialize Intel libraries and tools
+pytorch_extension = PyTorchExtension()
+tensorflow_extension = TensorFlowExtension()
+xgboost_optimizer = XGBoostOptimizer()
+modin_optimizer = ModinOptimizer()
+
+@app.get("/note_gen")
+async def summarize_s3_files(user:str):
+    user=user+"/"
+    bucket_name= "learnmateai"
+    folder_name= user+"Analysed_Notes"
+    try:
+        response = s3.list_objects_v2(Bucket=bucket_name, Prefix=folder_name)
+        for file in response['Contents']:
+            file_key = file['Key']
+            file_name = os.path.basename(file_key)
+            print(file_name)
+            summary = await summarize_file(bucket_name, file_key,file_name)
+            print(summary)
+            save_summary(file_name, summary,user)
+        return {'message': 'Created Notes and saved successfully.'}
+    except Exception as e:
+        return {'error': str(e)}
+
+async def summarize_file(bucket_name: str, file_key: str, file_name:str):
+    try:
+        file_name=file_name.split(".txt")[0]
+        response = s3.get_object(Bucket=bucket_name, Key=file_key)
+        file_content = response['Body'].read().decode('utf-8')
+        prompt = f'You are a teacher, make a full explanation for the topic: {file_name} below in good format. Include key concepts, explanations, and any relevant information. \nMake sure to cover these topics:\n{file_content}'
+        response = openai.ChatCompletion.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {
+                    "role": "user",
+                    "content": prompt
+                }
+            ]
+        )
+        summary = response.choices[0].message.content
+        return summary
+    except Exception as e:
+        raise e
+
+def save_summary(file_name: str, summary: str, user):
+    try:
+        file_name = file_name.split(".txt")[0]
+        save_key = f'{user}Notes_Topicwise/{file_name}.txt'
+        s3.put_object(Body=summary, Bucket=s3_bucket_name, Key=save_key)
+    except Exception as e:
+        raise e

Backend/Intel-oneAPI/Final_NotesChunker.py

@@ -0,0 +1,83 @@
+from fastapi import APIRouter
+import boto3
+import openai
+import time
+import intel.scikit_learn as skl
+import intel_pytorch_extension as ipex
+import intel_tensorflow_extension as ifex
+
+# Set up AWS S3 credentials
+s3_access_key = "YOUR_S3_ACCESS_KEY"
+s3_secret_access_key = "YOUR_S3_SECRET_ACCESS_KEY"
+s3_bucket_name = "YOUR_BUCKET_NAME"
+
+s3 = boto3.client("s3", aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_access_key)
+
+# Set up OpenAI API credentials
+openai.api_key = 'YOUR_OPENAI_API_KEY'
+
+def batch_text(input_text, delimiter="TOPIC:"):
+    batches = input_text.split(delimiter)
+    cleaned_batches = [batch.strip() for batch in batches if batch.strip()]
+    return cleaned_batches
+
+def upload_to_s3(bucket_name, folder_name, file_name, content):
+    s3 = boto3.client('s3', aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_access_key)
+    key = folder_name + '/' + file_name
+    s3.put_object(Body=content, Bucket=bucket_name, Key=key)
+
+app = APIRouter()
+
+@app.get("/process_files")
+def process_files(user: str):
+    user = user + "/"
+    
+    # Function to read and process a file
+    def process_file(file_name):
+        # Read file from S3
+        response = s3.get_object(Bucket=s3_bucket_name, Key=user + 'notes_txt/' + file_name)
+        file_content = response['Body'].read().decode('utf-8')
+
+        # Split file content into batches (adjust batch size as needed)
+        batch_size = 3000
+        batches = [file_content[i:i+batch_size] for i in range(0, len(file_content), batch_size)]
+
+        # Process batches
+        for batch in batches:
+            # Send batch to OpenAI API
+            response = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": f"divide the text topic wise (it should look like TOPIC:notes) notes should very breif and be created in a way so that you will be able to recreate the full txt :\n\n{batch}\n\n"
+                    }
+                ]
+            )
+
+            important_topics = response.choices[0].message.content
+            # Add a delay of 20 seconds to handle rate limit
+            time.sleep(20)
+
+            text_batches = batch_text(important_topics)
+
+            folder_name = f'{user}Analysed_Notes/{file_name.split(".")[0]}'
+
+            for i, batch in enumerate(text_batches):
+                lines = batch.split('\n')
+                file_name1 = lines[0].strip().replace(" ", "_") + '.txt'
+                content = '\n'.join(lines[1:]).strip()
+                upload_to_s3(s3_bucket_name, folder_name, file_name1, content)
+
+                # Print uploaded file information
+                print(f"File '{file_name1}' uploaded to '{s3_bucket_name}/{folder_name}'")
+
+    # Get the list of files in the "notes_txt" folder
+    response = s3.list_objects_v2(Bucket=s3_bucket_name, Prefix=user + 'notes_txt/')
+
+    # Process each file
+    for file in response['Contents']:
+        file_name = file['Key'].split('/')[-1]
+        process_file(file_name)
+
+    return {"message": "NOTES"}

Backend/Intel-oneAPI/Final_Notestotxt.py

@@ -0,0 +1,174 @@
+import os
+from fastapi import APIRouter, UploadFile, File, Form
+from pdf2image import convert_from_path
+from google.cloud import vision
+from typing import List
+import boto3
+from botocore.exceptions import NoCredentialsError
+from io import BytesIO
+import tempfile
+from openvino.inference_engine import IECore
+
+# Set up your S3 credentials and bucket name
+s3_access_key = "YOUR_S3_ACCESS_KEY"
+s3_secret_access_key = "YOUR_S3_SECRET_ACCESS_KEY"
+s3_bucket_name = "YOUR_S3_BUCKET_NAME"
+
+s3 = boto3.client(
+    "s3",
+    aws_access_key_id=s3_access_key,
+    aws_secret_access_key=s3_secret_access_key
+)
+
+ie = IECore()
+
+# Create an instance of APIRouter
+router = APIRouter()
+
+# Define your OpenVINO model paths
+model_xml = "PATH_TO_MODEL_XML"
+model_bin = "PATH_TO_MODEL_BIN"
+
+# Load the OpenVINO model
+net = ie.read_network(model=model_xml, weights=model_bin)
+exec_net = ie.load_network(network=net, device_name="CPU")
+
+# Define the input and output layer names of your model
+input_layer_name = "YOUR_INPUT_LAYER_NAME"
+output_layer_name = "YOUR_OUTPUT_LAYER_NAME"
+
+# Define any other necessary configuration or parameters
+
+# Rest of the code remains the same...
+# ...
+
+@router.post("/filestotext2")
+async def NotesToText_handler(user: str = Form(...)):
+    user = user + "/"
+    prefix = 'notes_pdf/'
+    prefix2 = 'pyqs_pdf/'
+    
+    # Delete existing files in the output folders
+    delete_folder_objects(user+'images/Notes_images/')
+    delete_folder_objects(user+'notes_txt/')
+    
+    convert(prefix, user)
+    convert(prefix2, user)
+    
+    return {"process completed"}
+
+
+def convert(prefix, user):
+    # List files in the S3 bucket with the specified prefix
+    response = s3.list_objects_v2(Bucket=s3_bucket_name, Prefix=user+prefix)
+    
+    # Extract the file names from the response
+    files = [obj['Key'] for obj in response.get('Contents', [])]
+    
+    # Process each file
+    for file_name in files:
+        file_name = os.path.splitext(os.path.basename(file_name))[0]
+        
+        print(f"Converting {file_name}....")
+        
+        # Delete existing files in the output folder
+        output_folder = f'{user}images/Notes_images/{file_name}'
+        delete_folder_objects(output_folder)
+        
+        # Download the PDF file from S3
+        pdf_object = s3.get_object(Bucket=s3_bucket_name, Key=f'{user}{prefix}{file_name}.pdf')
+        pdf_content = pdf_object['Body'].read()
+        
+        # Convert the PDF to images and save them in the output folder in S3
+        image_paths, noImg = pdf_to_images_from_bytes(pdf_content, output_folder, file_name)
+        print(noImg)
+        
+        os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'Files/client_file_vision.json'
+        client = vision.ImageAnnotatorClient()
+        
+        # [START vision_python_migration_text_detection]
+        image_contents = " "
+        
+        for j in range(noImg):
+            image_path = f'{output_folder}/page_{j+1}.jpeg'
+            
+            # Download the image from S3
+            image_object = s3.get_object(Bucket=s3_bucket_name, Key=image_path)
+            image_content = image_object['Body'].read()
+            
+            image_contents += image_content
+        
+        # Perform text detection using Google Cloud Vision API
+        response = client.text_detection(image=vision.Image(content=image_contents))
+        texts = response.text_annotations
+        
+        # Extract the detected text
+        detected_text = ""
+        for text in texts:
+            detected_text += text.description
+        
+        # Save the detected text in a text file
+        text_file_path = f'{user}notes_txt/{file_name}.txt'
+        upload_text_to_s3(detected_text, text_file_path)
+        
+        print(f"{file_name} converted.")
+    
+
+def pdf_to_images_from_bytes(pdf_bytes, output_folder, file_name):
+    images = convert_from_bytes(pdf_bytes)
+    image_paths = []
+    noImg = 0
+    
+    # Create the output folder if it doesn't exist
+    os.makedirs(output_folder, exist_ok=True)
+    
+    # Save each image as JPEG in the output folder
+    for i, image in enumerate(images):
+        image_path = f'{output_folder}/page_{i+1}.jpeg'
+        image.save(image_path, 'JPEG')
+        image_paths.append(image_path)
+        noImg += 1
+        
+    # Upload images to S3
+    upload_images_to_s3(image_paths, file_name)
+    
+    return image_paths, noImg
+
+
+def upload_images_to_s3(image_paths, file_name):
+    for image_path in image_paths:
+        with open(image_path, 'rb') as file:
+            try:
+                s3.upload_fileobj(file, s3_bucket_name, image_path)
+            except NoCredentialsError:
+                print("S3 credentials not available.")
+            except Exception as e:
+                print(f"Error uploading image to S3: {str(e)}")
+            finally:
+                # Remove the local image file
+                os.remove(image_path)
+
+
+def upload_text_to_s3(text, text_file_path):
+    try:
+        s3.put_object(Body=text, Bucket=s3_bucket_name, Key=text_file_path)
+    except NoCredentialsError:
+        print("S3 credentials not available.")
+    except Exception as e:
+        print(f"Error uploading text file to S3: {str(e)}")
+
+
+def delete_folder_objects(prefix):
+    # List objects in the S3 bucket with the specified prefix
+    response = s3.list_objects_v2(Bucket=s3_bucket_name, Prefix=prefix)
+    
+    # Extract the object keys from the response
+    objects = [obj["Key"] for obj in response.get("Contents", [])]
+    
+    # Delete each object
+    for obj_key in objects:
+        s3.delete_object(Bucket=s3_bucket_name, Key=obj_key)
+
+@router.get("/")
+async def hello():
+    return {"Byte 404 rocks"}