diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..dd84ea7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/custom.md b/.github/ISSUE_TEMPLATE/custom.md new file mode 100644 index 0000000..b894315 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/custom.md @@ -0,0 +1,8 @@ +--- +name: Custom issue template +about: Describe this issue template's purpose here. +title: '' +labels: '' +assignees: '' + +--- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 0000000..2a3f6b5 --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,41 @@ +name: Code Quality Checks + +on: + push: + paths: + - 'scrapegraphai/**' + - '.github/workflows/pylint.yml' + +jobs: + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install dependencies + run: uv sync --frozen + + - name: Run Ruff + run: uv run ruff check scrapegraphai + + - name: Run Black + run: uv run black --check scrapegraphai + + - name: Run isort + run: uv run isort --check-only scrapegraphai + + - name: Analysing the code with pylint + run: uv run poe pylint-ci + + - name: Check Pylint score + run: | + pylint_score=$(uv run poe pylint-score-ci | grep 'Raw metrics' | awk '{print $4}') + if (( $(echo "$pylint_score < 8" | bc -l) )); then + echo "Pylint score is below 8. Blocking commit." + exit 1 + else + echo "Pylint score is acceptable." + fi diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..001789c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,63 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + schedule: + - cron: '42 19 * * 5' + +jobs: + analyze: + name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..125c557 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,98 @@ +name: Release +on: + push: + branches: + - main + - pre/* + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Install git + run: | + sudo apt update + sudo apt install -y git + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install Node Env + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Checkout + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Build and validate package + run: | + uv venv + . .venv/bin/activate + uv pip install --upgrade setuptools wheel hatchling + uv sync --frozen + uv pip install -e . + uv build + uv pip install --upgrade pkginfo==1.12.0 twine==6.0.1 # Upgrade pkginfo and install twine + python -m twine check dist/* + + - name: Debug Dist Directory + run: ls -al dist + + - name: Cache build + uses: actions/cache@v3 + with: + path: ./dist + key: ${{ runner.os }}-build-${{ github.sha }} + + release: + name: Release + runs-on: ubuntu-latest + needs: build + environment: development + if: > + github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/pre/beta') || + (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged && + (github.event.pull_request.base.ref == 'main' || github.event.pull_request.base.ref == 'pre/beta')) + permissions: + contents: write + issues: write + pull-requests: write + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Restore build artifacts + uses: actions/cache@v3 + with: + path: ./dist + key: ${{ runner.os }}-build-${{ github.sha }} + + - name: Semantic Release + uses: cycjimmy/semantic-release-action@v4.1.0 + with: + semantic_version: 23 + extra_plugins: | + semantic-release-pypi@3 + @semantic-release/git + @semantic-release/commit-analyzer@12 + @semantic-release/release-notes-generator@13 + @semantic-release/github@10 + @semantic-release/changelog@6 + conventional-changelog-conventionalcommits@7 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6f18b2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.DS_Store +.venv +__pycache__ +*.pyc +*.pyo +*.pyd +*.pyw +*.pyz +*.pywz +*.pyzw +*.pyzwz \ No newline at end of file