Merge remote-tracking branch 'origin/yogesh-xxx-initial-commit' into yogesh-xxx-initial-commit

# Conflicts:
#	src/gaiaflow/managers/utils.py

Author: b-yogesh

.github/workflows/publish.yml

@@ -36,7 +36,7 @@ jobs:
 
       - name: Install and Test
         run:
-          pixi run test
+          pixi run test --cov=gaiaflow --cov-report=xml
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4

.github/workflows/unittest.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: Install and Test
         run:
-          pixi run test
+          pixi run test --cov=gaiaflow --cov-report=xml
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4

docs/dev_guide.md

@@ -99,10 +99,13 @@ data, models, and logging** (metrics, params, and artifacts to MLflow).
     - Learn how to use MLflow for experiment tracking.  
     - Perform inference on MLflow-logged models.  
 - If your data comes from S3 (hosted by BC), it’s best to **download a small 
-sample** and upload it to your local S3 storage (MinIO) for development and testing.  
+sample** and upload it to your local S3 storage (MinIO) for development and testing.
+_NOTE: This is recommended because there are egress-costs (costs that occur when 
+the data is pulled out from the AWS ecosystem) for everytime you pull the data._
 - For local datasets, upload them to MinIO as well. Later, when your workflow
 moves to production, ensure the data is available in S3.  
-`Talk to Tejas about uploading data to S3`
+`aws s3 sync /path/to/source /path/to/target --delete`
+Use `--delete` if you want the target to look exactly as the source
 ---
 
 #### 2. Refactor to Production Code
@@ -168,7 +171,7 @@ it inside a Docker environment:
 gaiaflow dev dockerize -p .
 ```
 
-- This builds an image for your package `Talk to Tejas`.
+- This builds an image for your package
 - Use the generated image name in the `image` parameter of `create_task`.
 - Pass environment variables via the `env_vars` parameter.
 - Set `mode = "dev_docker"` and trigger your workflow again.

docs/index.md

@@ -205,7 +205,8 @@ gaiaflow prod-local create-secret -p . --name <your-secret-name> --data SOME_KEY
      - Runs tasks as **Kubernetes pods in Minikube**.
      - Requires `image`
      - You can pass in `env_vars` and/or `secrets` if any. 
-     - For creating secrets in your minikube cluster, (coming soon, `Talk to Tejas`)
+     - For creating secrets in your minikube cluster _(coming soon)_
+
 
 Quick rule of thumb:
 

docs/prod_guide.md

@@ -142,21 +142,21 @@ You’ll need to configure the following secrets in your repository:
 
 ### 3. Task Secrets
 Any **secrets required by your tasks** must also be made available to the cluster running Airflow.  
-For this step, `Talk to Tejas`
-
+_(coming soon)_
 ---
 
-### 4. GitHub Release Workflow
-When you create a **GitHub release**, the following steps are triggered automatically:  
+### 4. GitHub Release
+
+To enable Airflow running in production to access your DAGs and your
+task packaged as an image, you must make a release following semantic versioning.
+
+To release your package, you can do it via Github UI.
+
+To make the release work successfully, you have to request the AWS access
+credentials from the infra team.
 
--  A Docker image is built and pushed to **AWS ECR**  
--  Your DAGs are uploaded to **S3**  
--  A **dispatch event** is sent to the CDR  
--  CDR pulls DAGs from S3  
--  Airflow reads DAGs from the CDR  
+Add this as a **GitHub repository secret** so your CI can use it.  
 
-For this to work, your team should provide you with an **AWS Role** that allows CI to push to ECR and S3.  
-Add this role as a **GitHub repository secret** so your CI can use it.  
 
 ---
 

pixi.lock

@@ -48,6 +48,7 @@ mkdocs = ">=1.6.1,<2"
 mkdocs-material = ">=9.6.17,<10"
 mkdocstrings = ">=0.30.0,<0.31"
 mkdocstrings-python = ">=1.17.0,<2"
+pytest-cov = ">=6.2.1,<7"
 
 [project.scripts]
 gaiaflow = "gaiaflow.cli.cli:app"

pyproject.toml

@@ -4,6 +4,8 @@
 import fsspec
 import typer
 
+from gaiaflow.constants import DEFAULT_IMAGE_NAME
+
 app = typer.Typer()
 fs = fsspec.filesystem("file")
 
@@ -103,6 +105,8 @@ def restart(
                   "minikube cluster.")
 def dockerize(
     project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
+    image_name: str = typer.Option(DEFAULT_IMAGE_NAME, "--image-name", "-i",
+                                   help=("Name of your image.")),
 ):
     imports = load_imports()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
@@ -117,6 +121,7 @@ def dockerize(
         user_project_path=user_project_path,
         action=imports.ExtendedAction.DOCKERIZE,
         local=False,
+        image_name=image_name
     )
 
 

src/gaiaflow/cli/commands/minikube.py

@@ -5,7 +5,7 @@
 import fsspec
 import typer
 
-from gaiaflow.constants import Service
+from gaiaflow.constants import Service, DEFAULT_IMAGE_NAME
 
 app = typer.Typer()
 fs = fsspec.filesystem("file")
@@ -240,6 +240,8 @@ def cleanup(
 @app.command(help="Containerize your package into a docker image locally.")
 def dockerize(
     project_path: Path = typer.Option(..., "--path", "-p", help="Path to your project"),
+    image_name: str = typer.Option(DEFAULT_IMAGE_NAME, "--image-name", "-i",
+                                   help=("Name of your image.")),
 ):
     imports = load_imports()
     gaiaflow_path, user_project_path = imports.create_gaiaflow_context_path(
@@ -260,6 +262,7 @@ def dockerize(
         user_project_path=user_project_path,
         action=imports.ExtendedAction.DOCKERIZE,
         local=True,
+        image_name=image_name
     )
 
 

src/gaiaflow/cli/commands/mlops.py

@@ -82,3 +82,5 @@ class Service(str, Enum):
         "limit_gpu": "1",
     },
 }
+
+DEFAULT_IMAGE_NAME = "user-image:v1"