Merge pull request #1 from bcdev/yogesh-xxx-initial-commit

Add Gaiaflow library v0.0.1.dev0

Author: b-yogesh

.github/workflows/gh_deploy.yml

@@ -0,0 +1,23 @@
+name: Deploy MkDocs to GitHub Pages
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install requirements
+        run: pip install mkdocs mkdocs-material mkdocstrings mkdocstrings-python mkdocs-jupyter
+
+      - name: Deploy to GitHub Pages
+        run: mkdocs gh-deploy --force

.github/workflows/publish.yml

@@ -0,0 +1,75 @@
+name: Package and Publish to PyPi
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  python-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: checkout xcube-clms
+        uses: actions/checkout@v4
+
+      - name: Install Pixi
+        run: curl -fsSL https://pixi.sh/install.sh | bash
+        shell: bash
+
+      - name: Add Pixi to PATH
+        run: echo "$HOME/.pixi/bin" >> $GITHUB_PATH
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: pixi add python=${{ matrix.python-version }} --no-install
+
+      - name: Install dependencies
+        run: pixi install
+
+      - name: Linting with ruff
+        run:
+          pixi run check
+
+      - name: Install and Test
+        run:
+          pixi run test --cov=gaiaflow --cov-report=xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+
+  PyPi-Deploy:
+    name: Publish Python Package to PyPI
+    runs-on: ubuntu-latest
+    needs: python-tests
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: |
+          python -m build
+
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          verbose: true

.github/workflows/unittest.yml

@@ -0,0 +1,43 @@
+name: Unittest Gaiaflow
+
+on:
+  push:
+
+jobs:
+  unittest:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: checkout xcube-clms
+        uses: actions/checkout@v4
+
+      - name: Install Pixi
+        run: curl -fsSL https://pixi.sh/install.sh | bash
+        shell: bash
+
+      - name: Add Pixi to PATH
+        run: echo "$HOME/.pixi/bin" >> $GITHUB_PATH
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: pixi add python=${{ matrix.python-version }} --no-install
+
+      - name: Install dependencies
+        run: pixi install
+
+      - name: Linting with ruff
+        run:
+          pixi run check
+
+      - name: Install and Test
+        run:
+          pixi run test --cov=gaiaflow --cov-report=xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -205,3 +205,5 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+# pixi environments
+.pixi

CHANGES.md

@@ -0,0 +1,10 @@
+## Version 0.0.1 (in development)
+
+This is an experimental version. A stable release will follow in the future.
+
+* First implementation of Gaiaflow CLI tool for managing local MLOps 
+infrastructure.
+* Moved all Docker files from the user’s cookiecutter project into this library.
+* A documentation site has been created.
+
+ 

MANIFEST.in

@@ -0,0 +1 @@
+include src/docker_stuff/*

README.md

@@ -1 +1,46 @@
-# gaiaflow
+# Gaiaflow
+
+[![Pixi Badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/prefix-dev/pixi/main/assets/badge/v0.json)](https://pixi.sh)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v0.json)](https://github.com/charliermarsh/ruff)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-blue)](https://bcdev.github.io/gaiaflow/)
+![Static Badge](https://img.shields.io/badge/Airflow-3.0-8A2BE2?logo=apacheairflow)
+![Static Badge](https://img.shields.io/badge/MLFlow-darkblue?logo=mlflow)
+![Static Badge](https://img.shields.io/badge/MinIO-red?logo=minio)
+![Static Badge](https://img.shields.io/badge/Jupyter-grey?logo=jupyter)
+![Static Badge](https://img.shields.io/badge/Minikube-lightblue?logo=kubernetes)
+
+
+Gaiaflow is a local-first MLOps infrastructure python package tool that simplifies the process 
+of building, testing, and deploying ML workflows.
+It provides an opinionated CLI for managing Airflow, MLflow, and other 
+dependencies, abstracting away complex configurations, and giving you 
+a smooth developer experience.
+
+Gaiaflow is a tool that
+- provides you with a local MLOps infrastructure via a CLI tool with 
+some prerequisites already installed.
+- handles the complex Airflow configuration and [Xcom](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/xcoms.html) 
+handling and provides the user a simpler interface for creating DAGs.
+- provides a [cookiecutter template](https://github.com/bcdev/gaiaflow-cookiecutter)
+to get started with your projects with a standardized structure.
+
+- provides tools to deploy models locally and in production (in future)
+- provides clear documentation on how to setup production environment to run your 
+workflows at scale (in future, private?)
+
+
+Prerequisites:
+- Docker
+- Docker compose
+- Miniforge
+- Mamba/Conda
+
+To install it, you can do it via:
+
+`pip install gaiaflow`
+
+Check installation:
+
+`gaiaflow --help`
+
+You can read the documentation [here]()

docs/cli_ref.md

@@ -0,0 +1,50 @@
+# CLI Reference
+
+To see options for any command, add `--help` at the end of your command.
+
+```bash
+gaiaflow --help
+```
+
+## `gaiaflow dev`
+
+Manage local development services.
+
+```bash
+gaiaflow dev [OPTIONS] COMMAND [ARGS]...
+```
+
+**Commands**
+
+- `start` – Start Gaiaflow dev services
+- `stop` – Stop Gaiaflow dev services
+- `restart` – Restart services
+- `cleanup` – Remove temporary context and state
+- `dockerize` – Build a Docker image of your ML package
+
+**Example**
+
+```bash
+gaiaflow dev start -p /path/to/project --service airflow --jupyter-port 8888
+```
+
+
+## `gaiaflow prod-local`
+Manage **production-like services** (via Minikube).
+
+**Commands**
+
+- `start` – Start production-like services
+- `stop` – Stop services
+- `restart` – Restart services
+- `dockerize` – Build image inside Minikube
+- `create-config` – Generate Airflow K8s config (debugging only)
+- `create-secret` – Create Kubernetes (K8s) secrets
+- `cleanup` – Remove Minikube-specific resources
+
+**Example**
+
+```bash
+gaiaflow prod-local start -p /path/to/project --force-new
+```
+