Merge branch 'pre/beta' into search_links_node

Author: VinciGit00

.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+      - pre/*
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install git
+        run: |
+          sudo apt update
+          sudo apt install -y git
+      - name: Install Python Env and Poetry
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - run: pip install poetry
+      - name: Install Node Env
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Build app
+        run: |
+          poetry install
+          poetry build
+        id: build_cache
+        if: success()
+      - name: Cache build
+        uses: actions/cache@v2
+        with:
+          path: ./dist
+          key: ${{ runner.os }}-build-${{ hashFiles('dist/**') }}
+        if: steps.build_cache.outputs.id != ''
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: build
+    environment: development
+    if: | 
+        github.event_name == 'push' && github.ref == 'refs/heads/main' || 
+        github.event_name == 'push' && github.ref == 'refs/heads/pre/beta' || 
+        github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged && github.event.pull_request.base.ref == 'main' || 
+        github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged && github.event.pull_request.base.ref == 'pre/beta'
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+      id-token: write
+    steps:
+      - name: Checkout repo
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Semantic Release
+        uses: cycjimmy/[email protected]
+        with:
+          semantic_version: 23
+          extra_plugins: |
+            semantic-release-pypi@3
+            @semantic-release/git 
+            @semantic-release/commit-analyzer@12 
+            @semantic-release/release-notes-generator@13 
+            @semantic-release/github@10 
+            @semantic-release/changelog@6
+            conventional-changelog-conventionalcommits@7
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.gitignore

@@ -35,3 +35,4 @@ poetry.lock
 
 # lock files
 *.lock
+poetry.lock

.releaserc.yml

@@ -0,0 +1,56 @@
+plugins:
+  - - "@semantic-release/commit-analyzer"
+    - preset: conventionalcommits
+  - - "@semantic-release/release-notes-generator"
+    - writerOpts:
+        commitsSort:
+        - subject
+        - scope
+      preset: conventionalcommits
+      presetConfig:
+        types:
+        - type: feat
+          section: Features
+        - type: fix
+          section: Bug Fixes
+        - type: chore
+          section: chore
+        - type: docs
+          section: Docs
+        - type: style
+          hidden: true
+        - type: refactor
+          section: Refactor
+        - type: perf
+          section: Perf
+        - type: test
+          section: Test
+        - type: build
+          section: Build
+        - type: ci
+          section: CI
+  - "@semantic-release/changelog"
+  - "semantic-release-pypi"
+  - "@semantic-release/github"
+  - - "@semantic-release/git"
+    - assets:
+        - CHANGELOG.md
+        - pyproject.toml
+      message: |-
+        ci(release): ${nextRelease.version} [skip ci]
+
+        ${nextRelease.notes}
+branches:
+  #child branches coming from tagged version for bugfix (1.1.x) or new features (1.x)
+  #maintenance branch
+  - name: "+([0-9])?(.{+([0-9]),x}).x"
+    channel: "stable"
+  #release a production version when merging towards main
+  - name: "main"
+    channel: "stable"
+  #prerelease branch
+  - name: "pre/beta"
+    channel: "dev"
+    prerelease: "beta"
+debug: true
+

CHANGELOG.md

@@ -0,0 +1,12 @@
+## [0.3.0-beta.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.2.8...v0.3.0-beta.1) (2024-04-26)
+
+
+### Features
+
+* trigger new beta release ([6f028c4](https://github.com/VinciGit00/Scrapegraph-ai/commit/6f028c499342655851044f54de2a8cc1b9b95697))
+
+
+### CI
+
+* add ci workflow to manage lib release with semantic-release ([92cd040](https://github.com/VinciGit00/Scrapegraph-ai/commit/92cd040dad8ba91a22515f3845f8dbb5f6a6939c))
+* remove pull request trigger and fix plugin release train ([876fe66](https://github.com/VinciGit00/Scrapegraph-ai/commit/876fe668d97adef3863446836b10a3c00a2eb82d))

CONTRIBUTING.md

@@ -26,7 +26,7 @@ To get started with contributing, follow these steps:
 
 ## Contributing Guidelines
 
-Please adhere to the following guidelines when contributing to AmazScraper:
+Please adhere to the following guidelines when contributing to ScrapeGraphAI:
 
 - Follow the code style and formatting guidelines specified in the [Code Style](#code-style) section.
 - Make sure your changes are well-documented and include any necessary updates to the project's documentation.
@@ -61,7 +61,7 @@ If you encounter any issues or have suggestions for improvements, please open an
 
 ## License
 
-AmazScraper is licensed under the **Apache License 2.0**. See the [LICENSE](LICENSE) file for more information.
+ScrapeGraphAI is licensed under the **MIT License**. See the [LICENSE](LICENSE) file for more information.
 By contributing to this project, you agree to license your contributions under the same license.
 
 Can't wait to see your contributions! :smile:

README.md

@@ -3,6 +3,7 @@
 [![Downloads](https://static.pepy.tech/badge/scrapegraphai)](https://pepy.tech/project/scrapegraphai)
 [![linting: pylint](https://img.shields.io/badge/linting-pylint-yellowgreen)](https://github.com/pylint-dev/pylint)
 [![Pylint](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/pylint.yml)
+[![CodeQL](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml/badge.svg)](https://github.com/VinciGit00/Scrapegraph-ai/actions/workflows/codeql.yml)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![](https://dcbadge.vercel.app/api/server/gkxQDAjfeX)](https://discord.gg/gkxQDAjfeX)
 
@@ -53,12 +54,11 @@ graph_config = {
         "model": "ollama/mistral",
         "temperature": 0,
         "format": "json",  # Ollama needs the format to be specified explicitly
-        "base_url": "http://localhost:11434",  # set Ollama URL arbitrarily
+        "base_url": "http://localhost:11434",  # set Ollama URL
     },
     "embeddings": {
         "model": "ollama/nomic-embed-text",
-        "temperature": 0,
-        "base_url": "http://localhost:11434",  # set Ollama URL arbitrarily
+        "base_url": "http://localhost:11434",  # set Ollama URL
     }
 }
 
@@ -79,7 +79,7 @@ print(result)
 Note: before using the local model remember to create the docker container!
 ```text
     docker-compose up -d
-    docker exec -it ollama ollama run stablelm-zephyr
+    docker exec -it ollama ollama pull stablelm-zephyr
 ```
 You can use which models avaiable on Ollama or your own model instead of stablelm-zephyr
 ```python

docs/source/index.rst

@@ -21,7 +21,7 @@ The following sections will guide you through the installation process and the u
    :caption: Getting Started
 
    getting_started/installation
-   getting_started/examples 
+   getting_started/examples
    modules/modules
 
 Indices and tables

examples/benchmarks/GenerateScraper/.env.example

@@ -1 +1 @@
-OPENAI_APIKEY="your openai api key"
+OPENAI_APIKEY="your openai key here"

examples/benchmarks/GenerateScraper/Readme.md

@@ -9,12 +9,14 @@ The time is measured in seconds
 
 The model runned for this benchmark is Mistral on Ollama with nomic-embed-text
 
+In particular, is tested with ScriptCreatorGraph
+
 | Hardware               | Model                                   | Example 1 | Example 2 |
 | ---------------------- | --------------------------------------- | --------- | --------- |
 | Macbook 14' m1 pro     | Mistral on Ollama with nomic-embed-text | 30.54s    | 35.76s    |
-| Macbook m2 max         | Mistral on Ollama with nomic-embed-text |           |           |
-| Macbook 14' m1 pro<br> | Llama3 on Ollama with nomic-embed-text  | 27.82s    | 29.986s   |
-| Macbook m2 max<br>     | Llama3 on Ollama with nomic-embed-text  |           |           |
+| Macbook m2 max         | Mistral on Ollama with nomic-embed-text | 18,46s    | 19.59     |
+| Macbook 14' m1 pro<br> | Llama3 on Ollama with nomic-embed-text  | 27.82s    | 29.98s    |
+| Macbook m2 max<br>     | Llama3 on Ollama with nomic-embed-text  | 20.83s    | 12.29s    |
 
 
 **Note**: the examples on Docker are not runned on other devices than the Macbook because the performance are to slow (10 times slower than Ollama). 
@@ -23,17 +25,17 @@ The model runned for this benchmark is Mistral on Ollama with nomic-embed-text
 **URL**: https://perinim.github.io/projects
 **Task**: List me all the projects with their description.
 
-| Name                | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
-| ------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- |
-| gpt-3.5-turbo       | 24.215268                | 1892         | 1802          | 90                | 1                   | 0.002883       |
-| gpt-4-turbo-preview | 6.614                    | 1936         | 1802          | 134               | 1                   | 0.02204        |
+| Name                | Execution time | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
+| ------------------- | ---------------| ------------ | ------------- | ----------------- | ------------------- | -------------- |
+| gpt-3.5-turbo       | 4.50s          | 1897         | 1802          | 95                | 1                   | 0.002893       |
+| gpt-4-turbo         | 7.88s          | 1920         | 1802          | 118               | 1                   | 0.02156        |
 
 ### Example 2: Wired
 **URL**: https://www.wired.com
 **Task**: List me all the articles with their description.
 
 | Name                | Execution time (seconds) | total_tokens | prompt_tokens | completion_tokens | successful_requests | total_cost_USD |
 | ------------------- | ------------------------ | ------------ | ------------- | ----------------- | ------------------- | -------------- |
-| gpt-3.5-turbo       |                          |              |               |                   |                     |                |
-| gpt-4-turbo-preview |                          |              |               |                   |                     |                |
+| gpt-3.5-turbo       |   Error (text too long)  |      -       |      -        |         -         |           -         |        -       |
+| gpt-4-turbo         |   Error (TPM limit reach)|      -       |      -        |         -         |           -         |        -       |
 

examples/benchmarks/GenerateScraper/benchmark_openai_gpt35.py

@@ -19,7 +19,7 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("GPT35_KEY")
+openai_key = os.getenv("OPENAI_APIKEY")
 
 graph_config = {
     "llm": {