feat: create scrapper function

Author: MatousMarik

advanced_tools_frameworks/web_scrapping_ai_agent/src/ai_scrapper_func.py

@@ -0,0 +1,43 @@
+# Import the required libraries
+# import streamlit as st
+from scrapegraphai.graphs import SmartScraperGraph
+
+from typing import Literal
+
+def ai_scrapper_func(openai_access_token: str, model: Literal["gpt-3.5-turbo", "gpt-4"], url: str, user_prompt: str) -> dict:
+    # Set up the Streamlit app
+    # st.title("Web Scrapping AI Agent 🕵️‍♂️")
+    # st.caption("This app allows you to scrape a website using OpenAI API")
+
+    # Get OpenAI API key from user
+    # openai_access_token = st.text_input("OpenAI API Key", type="password")
+
+    # if openai_access_token:
+    #     model = st.radio(
+    #         "Select the model",
+    #         ["gpt-3.5-turbo", "gpt-4"],
+    #         index=0,
+    #     )    
+    graph_config = {
+        "llm": {
+            "api_key": openai_access_token,
+            "model": f"openai/{model}",
+        },
+    }
+    # # Get the URL of the website to scrape
+    # url = st.text_input("Enter the URL of the website you want to scrape")
+    # # Get the user prompt
+    # user_prompt = st.text_input("What you want the AI agent to scrae from the website?")
+    
+    # Create a SmartScraperGraph object
+    smart_scraper_graph = SmartScraperGraph(
+        prompt=user_prompt,
+        source=url,
+        config=graph_config
+    )
+    # # Scrape the website
+    # if st.button("Scrape"):
+    #     result = smart_scraper_graph.run()
+    #     st.write(result)
+
+    return smart_scraper_graph.run()