Merge branch 'develop'

Author: Mdevpro78

.gitignore

@@ -190,3 +190,6 @@ _build/
 
 # Log files
 .logs
+
+# _version.py
+src/_version.py

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
   #########################################
   - repo: https://github.com/pre-commit/pre-commit-hooks
     # Standard collection of pre-commit hooks for various checks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: trailing-whitespace
         name: "✂️ whitespace · Remove trailing whitespace"
@@ -84,7 +84,7 @@ repos:
   #########################################
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff is a fast Python linter and formatter written in Rust
-    rev: v0.11.2
+    rev: v0.12.11
     hooks:
       - id: ruff
         name: "🐍 python · Format with Ruff"
@@ -99,7 +99,7 @@ repos:
   #########################################
   - repo: https://github.com/asottile/pyupgrade
     # Automatically upgrades Python syntax to newer versions
-    rev: v3.19.1
+    rev: v3.20.0
     hooks:
       - id: pyupgrade
         name: "🔄 python · Upgrade Python syntax"
@@ -108,7 +108,7 @@ repos:
 
   - repo: https://github.com/asottile/add-trailing-comma
     # Adds trailing commas to function calls and definitions
-    rev: v3.1.0
+    rev: v3.2.0
     hooks:
       - id: add-trailing-comma
         name: "🔤 python · Add trailing commas"
@@ -118,7 +118,7 @@ repos:
   # API Documentation Validation
   #########################################
   - repo: https://github.com/python-openapi/openapi-spec-validator
-    rev: 0.7.1
+    rev: 0.8.0b1
     hooks:
       - id: openapi-spec-validator
         name: "📝 api · Validate OpenAPI Spec"
@@ -180,7 +180,7 @@ repos:
 
   #########################################
   - repo: https://github.com/DavidAnson/markdownlint-cli2
-    rev: v0.17.2
+    rev: v0.18.1
     hooks:
       - id: markdownlint-cli2
         name: "📝 markdown · Lint markdown with markdownlint"
@@ -192,7 +192,7 @@ repos:
   # YAML File Validation
   #########################################
   - repo: https://github.com/adrienverge/yamllint.git
-    rev: v1.37.0
+    rev: v1.37.1
     hooks:
       - id: yamllint
         name: "📋 yaml · Lint YAML files with yamllint"
@@ -233,7 +233,7 @@ repos:
   # Scanning Hardcoded Secrets
   #########################################
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.24.2
+    rev: v8.28.0
     hooks:
       - id: gitleaks
         name: "🔒 security · Scan for hardcoded secrets"

Makefile

@@ -6,6 +6,11 @@
 	diff branch-from \
 	set-upstream-feature-branches
 
+ifneq ( ,$ (wildcard ./.env))
+include .env
+export
+endif
+
 # Docker Compose Makefile commands
 COMPOSE_FILE = docker-compose.yml
 # Build containers without using cache
@@ -49,13 +54,13 @@ uv_sync:
 	uv pip install -r requirements.txt
 # UV dependency management commands
 uv_sync_dev:
-	uv sync --extra dev
+	uv sync --group dev
 uv_sync_docs:
-	uv sync --extra docs
+	uv sync --group docs
 uv_sync_test:
-	uv sync --extra test
+	uv sync --group test
 uv_sync_lint:
-	uv sync --extra lint
+	uv sync --group lint
 uv_sync_all:
 	uv sync --python 3.12.10 --upgrade --all-groups --all-extras --reinstall --no-cache --resolution=highest
 uv_mkdocs_build:

README.md

@@ -1,38 +1,68 @@
-# MkDocForge
+<p align="center">
+<h1 align="center"> Sharif OCW Scrapy's Downloader </h1>
+<p align="center">
+
+<a href="https://img.shields.io">
+<img src="https://img.shields.io/badge/Project%20Status-Active-green.svg" alt="Project Status: Active">
+</a>
+
+<a href="https://www.python.org/downloads/">
+<img src="https://img.shields.io/badge/Python-3.11+-blue.svg" alt="Python Version">
+</a>
+
+<a href="https://scrapy.org">
+<img src="https://img.shields.io/badge/Scrapy-2.11+-green.svg" alt="Scrapy Version">
+</a>
 
+<a href="https://pydantic.dev">
+<img src="https://img.shields.io/badge/Pydantic-2.6+-blue.svg" alt="Pydantic Version">
+</a>
+
+<a href="https://www.mkdocs.org">
+<img src="https://img.shields.io/badge/docs-mkdocs-blue.svg" alt="Documentation">
+</a>
+
+<a href="LICENSE">
+<img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License">
+</a>
+
+</p>
+<p align="center"><em> One-week MVP sprint board for OCW Sharif Scrapy's Downloader.</em>
+</p>
 <p align="center">
-<a href="https://img.shields.io"><img src="https://img.shields.io/badge/Project%20Status-Active-green.svg" alt="Project Status: Active"></a>
-<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/Python-3.11+-blue.svg" alt="Python Version"></a>
-<a href="https://www.mkdocs.org"><img src="https://img.shields.io/badge/docs-mkdocs-blue.svg" alt="Documentation"></a>
-<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License"></a>
+<a href="#getting-started" class="md-button md-button--primary">Getting Started</a>
+<a href="https://github.com/Mdevpro78/sharif-ocw-scrapy-downloader" class="md-button">View on GitHub</a>
+</p>
 </p>
-
-<p align="center"><em>A comprehensive documentation platform powered by MkDocs with advanced features for technical documentation.</em></p>
 
 ## 📚 Overview
 
-MkDocForge is a powerful documentation platform built on top of MkDocs that provides enhanced features for creating beautiful, functional, and comprehensive technical documentation. It combines the simplicity of Markdown with advanced capabilities like versioned documentation, code documentation via Doxygen integration, diagrams, and a rich set of plugins to create professional documentation sites.
+**Project Goal:** Deliver an MVP Scrapy-based downloader for Sharif OCW that:
+
+- Fetches course metadata and sessions
+- Downloads all downloadable files
+- Organizes outputs into structured folders
+- Provides progress tracking and basic error handling
 
-## 🎯 Purpose
+**Success Criteria:**
 
-MkDocForge addresses common documentation challenges by providing:
+- Able to download at least one complete course (videos + PDFs)
+- Correct directory structure with sanitized filenames
+- Basic duplicate detection + retry handling works
+- GitHub issues, milestones, and PRs follow roadmap
 
-- **Unified Documentation**: Combine API references, user guides, architecture decisions, and more in one platform
-- **Versioned Documentation**: Track and maintain documentation across different software versions
-- **Code Integration**: Automatically generate API documentation from source code comments
-- **Rich Media Support**: Easily embed diagrams, images, and interactive elements
-- **Collaborative Workflow**: Git-based workflow for documentation-as-code practices
+**Team Size:** 1 developer (solo)
 
-## ✨ Key Features
+**Roles & Responsibilities:**
 
-- 🎨 **Material Design Theme**: Beautiful, responsive documentation with light/dark mode
-- 📊 **Diagram Support**: PlantUML integration for architectural and flow diagrams
-- 🔄 **Versioned Documentation**: Maintain docs for multiple software versions with Mike
-- 📝 **Markdown Extensions**: Enhanced markdown with admonitions, tabs, code annotations
-- 🧩 **Plugin Ecosystem**: Extensive collection of plugins for advanced functionality
-- 🐳 **Docker Support**: Containerized environment for consistent documentation builds
-- 🔍 **Full-Text Search**: Powerful search capabilities across documentation
-- 💻 **Code Documentation**: Doxygen integration for automatic code documentation
+- Developer: Implement, test, document, manage repo, and review
+
+**Definition of Done (DoD):**
+
+- Code compiles and runs without errors
+- Passes basic integration tests on one sample course
+- Artifacts stored in correct directory structure
+- Pull requests merged into `main` with review checklist passed
 
 ## 🚀 Getting Started
 
@@ -45,8 +75,8 @@ MkDocForge addresses common documentation challenges by providing:
 
 ```bash
 # Clone the repository
-git clone https://github.com/Mdevpro78/mkdocforge
-cd mkdocforge
+git clone https://github.com/Mdevpro78/sharif-ocw-scrapy-downloader/
+cd sharif-ocw-scrapy-downloader
 
 # Install dependencies using UV (recommended)
 make uv_sync_docs
@@ -97,22 +127,39 @@ Once running, access the documentation at [http://localhost:8000](http://localho
 
 ## 📁 Project Structure
 
-```
-.
-├── docs/                  # Documentation source files
-│   ├── architecture/      # Architecture decision records
-│   ├── blog/              # Blog posts
-│   ├── examples/          # Example configurations and code
-│   ├── git/               # Git workflow documentation
-│   ├── glossary/          # Project terminology
-│   ├── python/            # Python development guides
-│   └── ...                # Other documentation sections
-├── .github/               # GitHub workflows and CI/CD
-├── scripts/               # Utility scripts
-├── Dockerfile             # Docker configuration
-├── mkdocs.yml             # MkDocs configuration
-└── pyproject.toml         # Python project configuration
-```
+### Directories
+
+| Path     | Purpose                                     |
+| -------- | ------------------------------------------- |
+| docs/    | 📚 Docs: guidelines, roadmap, static assets |
+| src/     | 🧩 Source (Scrapy project + package)        |
+| .github/ | ⚙️ CI/CD workflows                          |
+| scripts/ | 🧰 Utility scripts                          |
+
+### Code layout (src)
+
+| Path                                     | Role                           |
+| ---------------------------------------- | ------------------------------ |
+| src/scrapy.cfg                           | Scrapy config                  |
+| src/sharif_ocw_downloader/config.py      | Configuration management       |
+| src/sharif_ocw_downloader/items.py       | Item definitions (data models) |
+| src/sharif_ocw_downloader/middlewares.py | Middleware components          |
+| src/sharif_ocw_downloader/pipelines.py   | Item pipelines (process/store) |
+| src/sharif_ocw_downloader/settings.py    | Scrapy settings                |
+| src/sharif_ocw_downloader/spiders/       | Spider implementations         |
+
+### Key files & configs
+
+| File                  | Purpose                        |
+| --------------------- | ------------------------------ |
+| Dockerfile            | 🐳 Build Docker image          |
+| docker-compose.yml    | Orchestrate services           |
+| Makefile              | Common automation tasks        |
+| mkdocs.yml            | MkDocs site config             |
+| pyproject.toml        | Python project metadata/config |
+| cliff.toml            | git-cliff (changelog) config   |
+| requirements.lock     | Locked production deps         |
+| requirements-dev.lock | Locked development deps        |
 
 ## 🔧 Configuration
 

docs/CODE_OF_CONDUCT.md

@@ -2,7 +2,7 @@
 
 # Code of Conduct
 
-> Our standards for participation in the DocForge community
+> Our standards for participation in the **Sharif OCW Scrapy's Downloader** community
 
 ## Our Pledge
 

docs/contributing.md

@@ -1,10 +1,10 @@
-# Contributing to DocForge
+# Contributing to Sharif OCW Scrapy's Downloader
 
 > Guidelines for contributing to this project
 
 ## 🚀 Getting Started
 
-Thank you for considering contributing to DocForge! This document outlines the process for contributing to the project and addresses common questions.
+Thank you for considering contributing to **Sharif OCW Scrapy's Downloader**! This document outlines the process for contributing to the project and addresses common questions.
 
 ## 📝 Code of Conduct
 
@@ -22,7 +22,13 @@ This project adheres to a Code of Conduct that all participants are expected to
     make setup_dev
     ```
 
-3. **Create a branch** for your changes:
+or
+
+```bash
+make uv_sync_all
+```
+
+1. **Create a branch** for your changes:
 
     ```bash
     git checkout -b feature/your-feature-name