Merge pull request #6 from labrijisaad/dockerize-the-code

Added the Docker version of the Pipeline

Author: labrijisaad

.dockerignore

@@ -0,0 +1,23 @@
+# ignore Dockerfile and .dockerignore
+Dockerfile
+.dockerignore
+
+# ignore potentially sensitive credentials files
+conf/**/*credentials*
+
+# ignore all local configuration
+conf/local
+!conf/local/.gitkeep
+
+# ignore everything in the following folders
+data
+logs
+notebooks
+references
+results
+
+# except the following
+!logs/.gitkeep
+!notebooks/.gitkeep
+!references/.gitkeep
+!results/.gitkeep

Dockerfile

@@ -0,0 +1,26 @@
+ARG BASE_IMAGE=python:3.9-slim
+FROM $BASE_IMAGE as runtime-environment
+
+# install project requirements
+COPY docker-requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache -r /tmp/requirements.txt && rm -f /tmp/requirements.txt
+
+# add kedro user
+ARG KEDRO_UID=999
+ARG KEDRO_GID=0
+RUN groupadd -f -g ${KEDRO_GID} kedro_group && \
+useradd -m -d /home/kedro_docker -s /bin/bash -g ${KEDRO_GID} -u ${KEDRO_UID} kedro_docker
+
+WORKDIR /home/kedro_docker
+USER kedro_docker
+
+FROM runtime-environment
+
+# copy the whole project except what is in .dockerignore
+ARG KEDRO_UID=999
+ARG KEDRO_GID=0
+COPY --chown=${KEDRO_UID}:${KEDRO_GID} . .
+
+EXPOSE 8888
+
+CMD ["kedro", "run"]

Makefile

@@ -1,29 +1,46 @@
+# Author information
 AUTHOR := Labriji Saad
 
-# Default target
+# Default target when no arguments are provided to make
 .DEFAULT_GOAL := help
 
-# run Jupyter Lab
+# Run Jupyter Lab - starts Jupyter Lab to allow for interactive development
 jupy:
+	@echo "Starting Jupyter Lab..."
 	@jupyter lab
 
-# run Kedro pipelines 
+# Run Kedro pipelines - executes the main pipeline defined in your Kedro project
 run:
+	@echo "Running Kedro pipeline..."
 	@kedro run
 
-# run Kedro Viz
+# Run Kedro Viz - launches Kedro's visualization tool to view the pipeline structure
 viz:
+	@echo "Running Kedro Viz..."
 	@kedro viz run
 
-# run Kedro Viz in autoreload mode
+# Run Kedro Viz in autoreload mode - automatically refreshes the visualization when changes are detected
 autoviz:
+	@echo "Running Kedro Viz in autoreload mode..."
 	@kedro viz run --autoreload
 
-# Display available make targets
+# Build Docker image for the project - creates a Docker image based on your Kedro project's specifications
+build:
+	@echo "Building Docker image..."
+	@kedro docker build
+
+# Run Kedro project inside a Docker container - executes the project within a Docker container
+dockerun:
+	@echo "Running Kedro project in Docker..."
+	@kedro docker run
+
+# Display help with available make targets
 help:
-	@echo  Available targets:
-	@echo    make jupy      - Activate the virtual environment and run Jupyter Lab
-	@echo    make run       - Run Kedro pipelines
-	@echo    make viz       - Run Kedro Viz
-	@echo    make autoviz   - Run Kedro Viz in autoreload mode
-	@echo  Author: $(AUTHOR)
+	@echo Available targets:
+	@echo   make jupy      - Activate the virtual environment and run Jupyter Lab
+	@echo   make run       - Run Kedro pipelines
+	@echo   make viz       - Run Kedro Viz
+	@echo   make autoviz   - Run Kedro Viz in autoreload mode
+	@echo   make build     - Build Docker image for the project
+	@echo   make dockerun  - Run Kedro project inside a Docker container
+	@echo Author: $(AUTHOR)

README.md

@@ -94,22 +94,46 @@ Kedro-Energy-Forecasting/
 │
 ├── .gitignore                                           # Untracked files to ignore
 ├── Makefile                                             # Set of tasks to be executed
+├── Dockerfile                                           # Instructions for building a Docker image
+├── .dockerignore                                        # Files and directories to ignore in Docker builds   
 ├── README.md                                            # Project documentation and setup guide
 └── requirements.txt                                     # Project dependencies
 ```
 
 ## 🚀 Getting Started
 
-Turn **raw CSV data** into a **trained pickle Machine Learning model** with these steps:
+First, **Clone the Repository** to download a copy of the code onto your local machine, and before diving into transforming **raw data** into a **trained pickle Machine Learning model**, please note:
 
-1. **Clone the Repository**: Download a copy of the code to your computer.
-2. **Set Up the Environment**: Create a virtual environment using Conda or venv.
-3. **Install Dependencies**: Run `pip install -r requirements.txt` in your environment to install the required libraries.
-4. **Run the Kedro Pipeline**: `make run` or `kedro run` – and witness magic 🪄
-5. **Review the Results**: After running the pipeline, look in the `04_reporting` and `05_model_output` directories to see your model's performance and results.
-6. **(Optional) Launch Kedro Viz**: To see a visual representation of your pipeline, run `make viz` or `kedro run viz`.
+🔴 **Important Preparation Steps**:
+- If you intend to run the code, it's better to remove the following directories if they exist: `data/02_processed`, `data/03_training_data`, `data/04_reporting`, and `data/05_model_output`. These directories will be regenerated or overwritten after executing the pipeline. They are **included** in the version control to **give you a preview of the expected outcomes**.
 
-_Need guidance on commands? Peek into the **Makefile** or use `kedro --help` for assistance._
+
+
+### Standard Method (Conda / venv) 🌿
+
+Adopt this method if you prefer a traditional Python development environment setup using Conda or venv.
+
+1. **Set Up the Environment**: Initialize a virtual environment with Conda or venv to isolate and manage your project's dependencies.
+   
+2. **Install Dependencies**: Inside your virtual environment, execute `pip install -r dev-requirements.txt` to install the necessary Python libraries.
+   
+3. **Run the Kedro Pipeline**: Trigger the pipeline processing by running `make run` or directly with `kedro run`. This step orchestrates your data transformation and modeling.
+   
+4. **Review the Results**: Inspect the `04_reporting` and `05_model_output` directories to assess the performance and outcomes of your models.
+   
+5. **(Optional) Explore with Kedro Viz**: To visually explore your pipeline's structure and data flows, initiate Kedro Viz using `make viz` or `kedro run viz`.
+
+### Docker Method 🐳
+
+Prefer this method for a containerized approach, ensuring a consistent development environment across different machines. Ensure Docker is operational on your system before you begin.
+
+1. **Build the Docker Image**: Construct your Docker image with `make build` or `kedro docker build`. This command leverages `dev-requirements.txt` for environment setup. For advanced configurations, see the [Kedro Docker Plugin Documentation](https://github.com/kedro-org/kedro-plugins/tree/main/kedro-docker).
+   
+2. **Run the Pipeline Inside a Container**: Execute the pipeline within Docker using `make dockerun` or `kedro docker run`. Kedro-Docker meticulously handles volume mappings to ensure seamless data integration between your local setup and the Docker environment.
+   
+3. **Access the Results**: Upon completion, the `04_reporting` and `05_model_output` directories will contain your model's reports and trained files, ready for review.
+
+For additional assistance or to explore more command options, refer to the **Makefile** or consult `kedro --help`.
 
 ## 🌐 Let's Connect!