diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..751bb90 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,53 @@ +venv/ +myenv/ +env/ +.venv/ + +__pycache__/ +*.py[cod] +*$py.class + +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +.sublime-* + +.git/ +.gitignore +.gitattributes + +.env +.env.local + +.coverage +htmlcov/ +.pytest_cache/ + +*.db +*.sqlite +*.sqlite3 +.DS_Store +Thumbs.db +*.log +.mypy_cache/ +.dmypy.json +dmypy.json +.pyre/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5c6d4cc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.10-slim +WORKDIR /app + +COPY app/requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . +CMD ["python", "-m", "app.main"] diff --git a/README.md b/README.md index 84e572a..a93cfe9 100644 --- a/README.md +++ b/README.md @@ -45,15 +45,74 @@ Contributors should search for these comments and fix the issues. ## ๐Ÿ›  Setup Instructions +### Local Setup + Clone the repo and install dependencies: ```bash git clone https://github.com//etl-problems.git cd etl-problems -pip install -r requirements.txt -python main.py +pip install -r app/requirements.txt +python -m app.main +``` + +### Running with Docker + +Containerize the ETL pipeline for a consistent, isolated development environment across all machines. + +**Prerequisites:** Docker and Docker Compose must be installed on your system. + +**Quick Start:** + +1. Clone the repository: +```bash +git clone https://github.com//etl-problems.git +cd etl-problems +``` + +2. Run the pipeline in a container: +```bash +docker-compose up +``` + +This command will: +- Build the Docker image from the provided `Dockerfile` +- Start the ETL pipeline in an isolated container +- Mount your local code directory as a volume, so changes you make to the code are immediately reflected in the container + +**Rebuilding the Image:** + +If you update dependencies in `requirements.txt`, rebuild the image: +```bash +docker-compose up --build +``` + +**Interactive Mode:** + +To run commands interactively inside the container: +```bash +docker-compose run etl bash ``` +Then inside the container, you can run: +```bash +python -m app.main +python -m pytest tests/ +``` + +**Stopping the Container:** + +```bash +docker-compose down +``` + +**Benefits:** +- ๐ŸŽฏ **Consistency**: Same environment for all developers (Python 3.10, all dependencies) +- ๐Ÿ“ฆ **Isolation**: No conflicts with local Python installations +- ๐Ÿš€ **Reproducibility**: Works the same on Windows, macOS, and Linux +- ๐Ÿ”ง **Hot Reload**: Code changes are immediately reflected without rebuilding +- ๐Ÿงช **Testing**: Run tests in an isolated environment + --- ## ๐Ÿงช Testing diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f0e5110 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3.8' + +services: + etl: + build: + context: . + dockerfile: Dockerfile + container_name: etl-pipeline + volumes: + - .:/app + environment: + - PYTHONUNBUFFERED=1 + working_dir: /app + command: python -m app.main