chore: release scripts and cleanup for monorepo (#2117)

Author: jjmachan

.github/workflows/ci.yaml

@@ -247,6 +247,8 @@ jobs:
       - name: Format check (dry run)
         run: |
           # Check if code is properly formatted (without making changes)
+          # Note: We use direct commands here instead of the standalone Makefiles
+          # to have precise control over CI-specific options like --check for dry-run
           echo "Checking ragas formatting..."
           black --check --config ragas/pyproject.toml ragas/src ragas/tests docs
           ruff check ragas/src docs ragas/tests

.github/workflows/python-publish.yml

@@ -1,12 +1,12 @@
-# This workflow will upload a Python Package using Twine when a release is created
+# This workflow will upload Python Packages using Twine when a release is created
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
 
 # This workflow uses actions that are not certified by GitHub.
 # They are provided by a third-party and are governed by
 # separate terms of service, privacy policy, and support
 # documentation.
 
-name: Upload Python Package
+name: Upload Python Packages
 
 on:
   release:
@@ -19,6 +19,15 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     environment: pypi-release
+    strategy:
+      matrix:
+        package:
+          - name: ragas
+            directory: ragas
+            token: PYPI_API_TOKEN
+          - name: ragas_experimental
+            directory: experimental
+            token: PYPI_API_TOKEN_EXPERIMENTAL
     steps:
     - uses: actions/checkout@v3
       with:
@@ -33,10 +42,13 @@ jobs:
         pip install --upgrade setuptools setuptools_scm[toml] build 
     - name: get setuptools-scm version 
       run: python -m setuptools_scm
+      working-directory: ${{ matrix.package.directory }}
     - name: Build package
       run: python -m build
+      working-directory: ${{ matrix.package.directory }}
     - name: Publish package
       uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
       with:
         user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
+        password: ${{ secrets[matrix.package.token] }}
+        packages-dir: ${{ matrix.package.directory }}/dist/

CLAUDE.md

@@ -29,92 +29,98 @@ pip install -e "./experimental[dev]"
 
 ## Common Commands
 
-### Build and Development
+### Monorepo-Wide Commands (from root directory)
 
 ```bash
-# Format code (ragas core)
-make format
+# Setup and installation
+make install        # Install dependencies for both projects
 
-# Format code (experimental)
-make format-experimental 
+# Code quality (runs on both ragas/ and experimental/)
+make format         # Format and lint all code
+make type           # Type check all code
+make check          # Quick health check (format + type, no tests)
 
-# Format all code in the monorepo
-make format-all
+# Testing
+make test           # Run all unit tests
+make test-e2e       # Run end-to-end tests
 
-# Lint code (ragas core)
-make lint
+# CI/Build
+make run-ci         # Run complete CI pipeline for both projects
+make clean          # Clean all generated files
 
-# Lint code (experimental)
-make lint-experimental
+# Documentation
+make build-docs     # Build all documentation
+make serve-docs     # Serve documentation locally
 
-# Lint all code in the monorepo
-make lint-all
-
-# Type check code (ragas core)
-make type
-
-# Type check code (experimental)
-make type-experimental
-
-# Type check all code in the monorepo
-make type-all
+# Benchmarks
+make benchmarks     # Run performance benchmarks
+make benchmarks-docker # Run benchmarks in Docker
+```
 
-# Run all CI checks for ragas core
-make run-ci
+### Project-Specific Commands
 
-# Run all CI checks for experimental
-make run-ci-experimental
+Each project directory (`ragas/` and `experimental/`) has its own Makefile with core development commands:
 
-# Run all CI checks for both projects
-make run-ci-all
+```bash
+# Ragas core development (from ragas/ directory)
+cd ragas
+make format         # Format ragas code only
+make type           # Type check ragas code only
+make check          # Quick format + type check
+make test           # Run ragas tests only
+make run-ci         # Run ragas CI pipeline only
+
+# Experimental development (from experimental/ directory)
+cd experimental
+make format         # Format experimental code only
+make type           # Type check experimental code only
+make check          # Quick format + type check
+make test           # Run experimental tests only
+make run-ci         # Run experimental CI pipeline only
 ```
 
 ### Testing
 
 ```bash
-# Run ragas core tests
+# Run all tests in the monorepo (from root)
 make test
 
 # Run specific test (using pytest -k flag)
 make test k="test_name"
 
-# Run ragas end-to-end tests
+# Run end-to-end tests
 make test-e2e
 
-# Run experimental tests
-make test-experimental
+# Run tests for specific projects
+cd ragas && make test           # Run ragas tests only
+cd experimental && make test    # Run experimental tests only
 
-# Run all tests in the monorepo
-make test-all
+# Direct pytest commands for more control
+cd ragas && uv run pytest tests/unit -k "test_name"
+cd experimental && uv run pytest -v
 ```
 
 ### Documentation
 
 ```bash
-# Build ragas documentation
-make build-docsite-ragas
-
-# Build experimental documentation
-make build-docsite-experimental
-
-# Build all documentation
-make build-docsite
+# Build all documentation (from root)
+make build-docs
 
 # Serve documentation locally
-make serve-docsite
+make serve-docs
+
+# Process experimental notebooks
+make process-experimental-notebooks
 ```
 
 ### Benchmarks
 
 ```bash
-# Run benchmarks for Evaluation
-make run-benchmarks-eval
-
-# Run benchmarks for TestSet Generation
-make run-benchmarks-testset
+# Run all benchmarks locally
+make benchmarks
 
-# Run benchmarks in docker
-make run-benchmarks-in-docker
+# Run benchmarks in Docker
+make benchmarks-docker
 ```
 
 ## Project Architecture

DEVELOPMENT.md

@@ -157,10 +157,22 @@ make run-ci          # Run full CI pipeline
 
 ### Working with Specific Projects
 
+Each project directory (`ragas/` and `experimental/`) now has its own standalone Makefile with core development commands. You can work directly within each project directory using these local Makefiles.
+
 #### Ragas Core Development
 ```bash
 # Navigate to the ragas directory for project-specific work
 cd ragas
+
+# Use the local Makefile for development
+make help           # See available commands
+make format         # Format ragas code only
+make type           # Type check ragas code only
+make test           # Run ragas tests only
+make check          # Quick format + type check
+make run-ci         # Run full ragas CI pipeline
+
+# Or use direct commands for specific tasks
 uv run pytest tests/unit          # Run specific tests
 uv run pyright src               # Type check specific code
 ```
@@ -169,9 +181,28 @@ uv run pyright src               # Type check specific code
 ```bash
 # Navigate to experimental directory
 cd experimental  
+
+# Use the local Makefile for development
+make help           # See available commands
+make format         # Format experimental code only
+make type           # Type check experimental code only
+make test           # Run experimental tests only
+make check          # Quick format + type check
+make run-ci         # Run full experimental CI pipeline
+
+# Or use direct commands
 uv run pytest                   # Run experimental tests
 ```
 
+#### Monorepo-Wide Development
+```bash
+# From the root directory, commands operate on both projects
+make format         # Format code in both ragas/ and experimental/
+make type           # Type check both projects
+make test           # Run all tests in both projects
+make run-ci         # Run full CI for both projects
+```
+
 ## Testing Strategy
 
 ### Test Types
@@ -344,6 +375,10 @@ ls -la  # Check current directory structure
 make format  # After making changes
 make test    # Verify functionality
 
+# For project-specific work
+cd ragas && make help           # See ragas-specific commands
+cd experimental && make help    # See experimental-specific commands
+
 # For investigation
 uv run pytest --collect-only  # See available tests
 uv run ruff check --no-fix    # Check issues without fixing

Makefile

@@ -3,7 +3,7 @@ GIT_ROOT ?= $(shell git rev-parse --show-toplevel)
 # Optionally show commands being executed with V=1
 Q := $(if $(V),,@)
 
-# Common paths
+# Common paths - only used for monorepo-wide operations
 RAGAS_PATHS := ragas/src ragas/tests docs
 EXPERIMENTAL_PATH := experimental/ragas_experimental
 
@@ -21,46 +21,30 @@ install: ## Install dependencies for both ragas and experimental
 	@echo "Installing experimental dependencies..."
 	$(Q)uv pip install -e "./experimental[dev]"
 
-setup: install ## Complete development environment setup
-	@echo "Development environment setup complete!"
-	@echo "Available commands: make help"
-
 # =============================================================================
 # CODE QUALITY
 # =============================================================================
 
 .PHONY: help install setup format type check clean test test-e2e benchmarks benchmarks-docker run-ci run-ci-fast run-ci-format-check run-ci-type run-ci-tests build-docs serve-docs process-experimental-notebooks
 format: ## Format and lint all code in the monorepo
 	@echo "Formatting and linting all code..."
-	@echo "(black) Formatting ragas..."
-	$(Q)uv run black --config ragas/pyproject.toml $(RAGAS_PATHS)
-	@echo "(black) Formatting stubs..."
-	$(Q)find ragas/src -name "*.pyi" ! -name "*_pb2*" -exec uv run black --pyi --config ragas/pyproject.toml {} \;
-	@echo "(ruff) Auto-fixing ragas (includes import sorting and unused imports)..."
-	$(Q)uv run ruff check $(RAGAS_PATHS) --fix-only
-	@echo "(ruff) Final linting check for ragas..."
-	$(Q)uv run ruff check $(RAGAS_PATHS)
-	@echo "(black) Formatting experimental..."
-	$(Q)cd experimental && uv run black ragas_experimental
-	@echo "(ruff) Auto-fixing experimental (includes import sorting and unused imports)..."
-	$(Q)cd experimental && uv run ruff check ragas_experimental --fix-only
-	@echo "(ruff) Final linting check for experimental..."
-	$(Q)cd experimental && uv run ruff check ragas_experimental
+	@echo "Formatting ragas..."
+	$(Q)$(MAKE) -C ragas format
+	@echo "Formatting experimental..."
+	$(Q)$(MAKE) -C experimental format
 
 type: ## Type check all code in the monorepo
 	@echo "Type checking all code..."
-	@echo "(pyright) Typechecking ragas..."
-	$(Q)cd ragas && PYRIGHT_PYTHON_FORCE_VERSION=latest pyright src
-	@echo "(pyright) Typechecking experimental..."
-	# TODO: Fix experimental type checking for 0.3 release - currently has 96 type errors
-	# $(Q)PYRIGHT_PYTHON_FORCE_VERSION=latest pyright $(EXPERIMENTAL_PATH)
-	@echo "Experimental type checking temporarily disabled - TODO: fix for 0.3 release"
+	@echo "Type checking ragas..."
+	$(Q)$(MAKE) -C ragas type
+	@echo "Type checking experimental..."
+	$(Q)$(MAKE) -C experimental type
 
 check: format type ## Quick health check (format + type, no tests)
 	@echo "Code quality check complete!"
 
 # =============================================================================
-# TESTING
+# BENCHMARKS
 # =============================================================================
 benchmarks: ## Run all benchmarks locally
 	@echo "Running all benchmarks..."
@@ -79,7 +63,12 @@ benchmarks-docker: ## Run benchmarks in docker
 # CI/BUILD
 # =============================================================================
 
-run-ci: run-ci-format-check run-ci-type run-ci-tests ## Run complete CI pipeline (mirrors GitHub CI exactly)
+run-ci: ## Run complete CI pipeline (mirrors GitHub CI exactly)
+	@echo "Running complete CI pipeline..."
+	@echo "Running ragas CI..."
+	$(Q)$(MAKE) -C ragas run-ci
+	@echo "Running experimental CI..."
+	$(Q)$(MAKE) -C experimental run-ci
 	@echo "All CI checks passed!"
 
 run-ci-format-check: ## Run format check in dry-run mode (like GitHub CI)
@@ -118,15 +107,15 @@ clean: ## Clean all generated files
 	$(Q)find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
 
 # =============================================================================
-# DOCUMENTATION
+# TESTING
 # =============================================================================
 
 test: ## Run all unit tests in the monorepo
 	@echo "Running all unit tests..."
 	@echo "Running ragas tests..."
-	$(Q)cd ragas && uv run pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
+	$(Q)$(MAKE) -C ragas test $(shell if [ -n "$(k)" ]; then echo "k=$(k)"; fi)
 	@echo "Running experimental tests..."
-	$(Q)cd experimental && uv run pytest
+	$(Q)$(MAKE) -C experimental test
 
 test-e2e: ## Run all end-to-end tests
 	@echo "Running all end-to-end tests..."
@@ -141,7 +130,7 @@ test-e2e: ## Run all end-to-end tests
 	fi
 
 # =============================================================================
-# BENCHMARKS
+# DOCUMENTATION
 # =============================================================================
 
 process-experimental-notebooks: ## Process experimental notebooks to markdown for MkDocs