latestpyqsorter done

Author: ANGELOANTU7

Backend/LatestSorter.py

@@ -0,0 +1,146 @@
+from fastapi import APIRouter
+import boto3
+import openai
+import time
+from botocore.exceptions import ClientError
+
+number=4
+s3_access_key = "AKIAZTHHIOR4JJ5HLTUB"
+s3_secret_access_key = "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"
+s3_bucket_name = "learnmateai"
+
+s3 = boto3.client("s3", aws_access_key_id=s3_access_key, aws_secret_access_key=s3_secret_access_key)
+
+# Set up OpenAI API credentials
+openai.api_key = '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'
+
+def batch_text(input_text, delimiter="Module"):
+    batches = input_text.split(delimiter)
+    cleaned_batches = [batch.strip() for batch in batches if batch.strip()]
+    if(len(cleaned_batches)<3):
+        batches = input_text.split("MODULE")
+        cleaned_batches = [batch.strip() for batch in batches if batch.strip()]
+    return cleaned_batches
+
+def upload_to_s3(bucket_name, folder_name, file_name, content):
+    s3 = boto3.client('s3')
+    key = folder_name + '/' + file_name
+    s3.put_object(Body=content, Bucket=bucket_name, Key=key)
+    
+def get_text_from_s3(bucket_name, file_name, encoding='utf-8'):
+    response = s3.get_object(Bucket=bucket_name, Key=file_name)
+    content = response['Body'].read()
+    
+    try:
+        text_content = content.decode(encoding)
+    except UnicodeDecodeError:
+        # Handle decoding error gracefully
+        text_content = content.decode('latin-1')  # Try an alternative encoding
+    
+    return text_content
+
+app = APIRouter()
+
+@app.get("/sorter")
+def process_files():
+    # Function to read and process a file
+    def process_file(file_name):
+        # Read file from S3
+        
+        response = s3.get_object(Bucket='learnmateai', Key='pyqs_txt/' + file_name)
+        file_content = response['Body'].read().decode('utf-16-le')
+
+        # Split file content into batches (adjust batch size as needed)
+        batch_size = 30000
+        batches = [file_content[i:i+batch_size] for i in range(0, len(file_content), batch_size)]
+        response2 = s3.get_object(Bucket='learnmateai', Key= "syllabus_pdf/syllabus.txt")
+        topics = response2['Body'].read().decode('utf-8')
+        # Process batches
+        Sorted_PYQ_Mod=[[]for _ in range(5)]
+        for batch in batches:
+            # Send batch to OpenAI API
+
+            
+            response = openai.ChatCompletion.create(
+                model="gpt-3.5-turbo",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": f"I will feed you a question paper as text,sort the question in the text below based on this syllabus having {number} modules :{topics} (it should look exactly like MODULE:questions ) all questions should cluster under its module , the output should exactly  have the {number} number of ""MODULE"" written under each the questions come ,it should have all MODULE even if any is empty, never give question with there modules i need it as grouped under module always :\n\n{batch}\n\n"
+                    }
+                ]
+            )
+
+            important_topics = response.choices[0].message.content
+            #print(important_topics)
+            #return important_topics
+            # Add a delay of 20 seconds to handle rate limit
+            
+            
+            text_batches = batch_text(important_topics)
+            #print(text_batches)
+            
+            bucket_name = 'learnmateai'
+            folder_name = 'Sorted_PYQS/'
+            
+            
+            i=0
+            try:
+                
+                for batch in enumerate(text_batches):
+                    print(batch)
+                    
+                    result=' '.join(str(element) for element in batch)
+                    new_content = result
+                    response = s3.get_object(Bucket=bucket_name, Key=folder_name+"Module"+str(i+1)+".txt")
+                    current_content = response['Body'].read().decode('utf-8')
+
+
+                    updated_content = current_content + new_content
+
+                    # Upload the updated content to S3
+                    s3.put_object(Bucket=bucket_name, Key=folder_name+"Module"+str(i+1)+".txt", Body=updated_content.encode('utf-8'))
+
+                    # Print uploaded file information
+                    print(f"File  uploaded to '{bucket_name}/{folder_name}'")
+                    i=i+1
+                    
+                
+                
+                time.sleep(20)
+
+            except ClientError as e:
+                
+                if e.response['Error']['Code'] == 'NoSuchKey':
+                    print("File not found in S3 bucket.")
+                    
+                    for batch in enumerate(text_batches):
+                        print(batch)
+                        
+                        result=' '.join(str(element) for element in batch)
+                        new_content = '\n'.join(result[1:]).strip()
+
+
+                        updated_content =new_content
+
+                        # Upload the updated content to S3
+                        s3.put_object(Bucket=bucket_name, Key=folder_name+"Module"+str(i+1)+".txt", Body=updated_content.encode('utf-8'))
+
+                        # Print uploaded file information
+                        print(f"File  uploaded to '{bucket_name}/{folder_name}'")
+                        i=i+1
+                else:
+                    print("An error occurred:", e)
+                
+
+
+
+    # Get the list of files in the "notes_txt" folder
+    response = s3.list_objects_v2(Bucket='learnmateai', Prefix='pyqs_txt/')
+
+    # Process each file
+    for file in response['Contents']:
+        file_name = file['Key'].split('/')[-1]
+        process_file(file_name)
+
+    return {"message": "File processing completed."}

Backend/__pycache__/LatestSorter.cpython-310.pyc

@@ -2,7 +2,7 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from mangum import Mangum
-#from Backend.pyqsorter import router as api1_router
+from Backend.LatestSorter import app as sorter
 #from Backend.summariser import router_summariser as summariser
 
 #from Backend.Notes_Analyser import router as api4_router
@@ -30,7 +30,7 @@
 # Mount the API routerss
 #app.include_router(api1_router)
 
-#app.include_router(sorter)
+app.include_router(sorter)
 #app.include_router(api4_router)
 app.include_router(chunker)
 #app.include_router(notestotxt)