Merge pull request #86 from neo4j-labs/create_source_node_youtube

Youtube option Support

Author: prakriti-solankey

backend/requirements.txt

@@ -180,3 +180,5 @@ zipp==3.17.0
 poppler-utils
 pytesseract
 pdf2image
+youtube-transcript-api
+pytube

backend/score.py

@@ -11,15 +11,12 @@ def healthy_condition():
     output = {"healthy": True}
     return output
 
-
 def healthy():
     return True
 
-
 def sick():
     return False
 
-
 app = FastAPI()
 
 app.add_middleware(
@@ -31,10 +28,9 @@ def sick():
 )
 app.add_api_route("/health", health([healthy_condition, healthy]))
 
-
 @app.post("/sources")
 async def create_source_knowledge_graph(
-    uri=Form(), userName=Form(), password=Form(), file: UploadFile = File(...)
+    uri=Form(), database=Form(), userName=Form(), password=Form(), file: UploadFile = File(...), model=Form()
 ):
     """
     Calls 'create_source_node_graph' function in a new thread to create
@@ -49,42 +45,42 @@ async def create_source_knowledge_graph(
     Returns:
          'Source' Node creation in Neo4j database
     """
-    try:
-        result = await asyncio.to_thread(
-            create_source_node_graph, uri, userName, password, file
-        )
-        return result
-    except Exception as e:
-        job_status = "Failure"
-        error_message = str(e)
-        logging.exception(f"Exception Stack trace:{e}")
-        return create_api_response(job_status, error=error_message)
-
+    result = await asyncio.to_thread(
+        create_source_node_graph_local_file, uri, database, userName, password, file, model
+    )
+    return result
 
-@app.post("/bucket/scan")
-async def create_source_knowledge_graph(
+@app.post("/url/scan")
+async def create_source_knowledge_graph_url(
     uri=Form(),
+    database=Form(),
     userName=Form(),
     password=Form(),
-    s3_url_dir=Form(),
+    source_url=Form(),
     aws_access_key_id=Form(None),
     aws_secret_access_key=Form(None),
+    max_limit=Form(5),
+    query_source=Form(None),
+    model=Form(None)
 ):
-    return create_source_node_graph_s3(
-        uri, userName, password, s3_url_dir, aws_access_key_id, aws_secret_access_key
+    return create_source_node_graph_url(
+        uri, database, userName, password, source_url, max_limit, query_source, model, aws_access_key_id, aws_secret_access_key
     )
 
 
 @app.post("/extract")
 async def extract_knowledge_graph_from_file(
     uri=Form(),
+    database=Form(),
     userName=Form(),
     password=Form(),
     file: UploadFile = File(None),
     model=Form(),
-    s3_url=Form(None),
+    source_url=Form(None),
     aws_access_key_id=Form(None),
     aws_secret_access_key=Form(None),
+    wiki_query=Form(None),
+    max_sources=Form(None),
 ):
     """
     Calls 'extract_graph_from_file' in a new thread to create Neo4jGraph from a
@@ -100,65 +96,54 @@ async def extract_knowledge_graph_from_file(
     Returns:
           Nodes and Relations created in Neo4j databse for the pdf file
     """
-    try:
-        if file:
-            return await asyncio.to_thread(
-                extract_graph_from_file,
-                uri,
-                userName,
-                password,
-                model,
-                file=file,
-                s3_url=None,
-            )
-        elif s3_url:
-            return await asyncio.to_thread(
-                extract_graph_from_file,
-                uri,
-                userName,
-                password,
-                model,
-                s3_url=s3_url,
-                aws_access_key_id=aws_access_key_id,
-                aws_secret_access_key=aws_secret_access_key,
-            )
-        else:
-            return {"job_status": "Failure", "error": "No file found"}
-    except Exception as e:
-        job_status = "Failure"
-        error_message = str(e)
-        logging.exception(f"Exception Stack trace:{e}")
-        return create_api_response(job_status, error=error_message)
-
+    
+    if file:
+        return await asyncio.to_thread(
+            extract_graph_from_file,
+            uri,
+            database,
+            userName,
+            password,
+            model,
+            file=file,
+            source_url=None,
+            wiki_query=wiki_query,
+            max_sources=max_sources,
+        )
+    elif source_url:
+        return await asyncio.to_thread(
+            extract_graph_from_file,
+            uri,
+            database,
+            userName,
+            password,
+            model,
+            source_url=source_url,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            wiki_query=wiki_query,
+            max_sources=max_sources,
+        )
+    else:
+        return {"job_status": "Failure", "error": "No file found"}
+    
 
 @app.get("/sources_list")
-async def get_source_list():
+async def get_source_list(uri=Form(),database=Form(),userName=Form(),password=Form()):
     """
     Calls 'get_source_list_from_graph' which returns list of sources which alreday exist in databse
     """
-    try:
-        result = await asyncio.to_thread(get_source_list_from_graph)
-        return result
-    except Exception as e:
-        job_status = "Failure"
-        error_message = str(e)
-        logging.exception(f"Exception Stack trace:{e}")
-        return create_api_response(job_status, error=error_message)
+    result = await asyncio.to_thread(get_source_list_from_graph,uri,database,userName,password)
+    return result
 
 @app.post("/update_similarity_graph")
 async def update_similarity_graph():
     """
     Calls 'update_graph' which post the query to update the similiar nodes in the graph
     """
-    try:
-        result = await asyncio.to_thread(update_graph)
-        return result
-    except Exception as e:
-        job_status = "Failure"
-        error_message = str(e)
-        logging.exception(f"Exception Stack trace:{e}")
-        return create_api_response(job_status, error=error_message)
-
-
+    
+    result = await asyncio.to_thread(update_graph)
+    return result
+    
 if __name__ == "__main__":
-    uvicorn.run(app)
+    uvicorn.run(app)

backend/src/diffbot_transformer.py

@@ -31,10 +31,14 @@ def extract_graph_from_diffbot(graph: Neo4jGraph,
     graph_document_list = []
 
     logging.info(f"create relationship between source,chunk and entity nodes created from Diffbot")
-    for chunk in chunks:
+    for i,chunk in enumerate(chunks):
+        if i == 0:
+            firstChunk = True
+        else:
+            firstChunk = False
         graph_document = diffbot_nlp.convert_to_graph_documents([chunk])
         graph.add_graph_documents(graph_document)
-        create_source_chunk_entity_relationship(file_name,graph,graph_document,chunk,uri,userName,password)
+        create_source_chunk_entity_relationship(file_name,graph,graph_document,chunk,uri,userName,password,firstChunk)
         graph_document_list.append(graph_document[0]) 
 
     graph.refresh_schema()