Add support for source and target schemas in migration commands, enhance CLI options for better flexibility

Author: Domains18

README.md

@@ -1,17 +1,140 @@
-# ================================
-# README.md
 # CloudSQL to Supabase Migration Tool
 
-A CLI tool to migrate PostgreSQL databases from Google Cloud SQL to Supabase.
+A robust CLI tool to safely migrate PostgreSQL databases from Google Cloud SQL to Supabase.
+
+
+## Installation
+
+### 1. Clone the repository (or create the project structure)
+
+```bash
+git clone https://github.com/Domains18/migrate-cloud-supabase.git
+cd cloudsql-to-supabase
+```
+
+### 2. Set up a virtual environment (recommended)
+
+```bash
+python -m venv venv
+source venv/bin/activate  # On Windows: venv\Scripts\activate
+```
+
+### 3. Install the package
+
+```bash
+pip install -e .
+```
+
+### 4. Configure your environment
+
+Copy the example `.env` file and edit it with your database details:
+
+```bash
+cp .env.example .env
+```
+
+Edit the `.env` file with your database credentials:
+
+```env
+# CloudSQL Configuration
+CLOUDSQL_USER=your_cloudsql_user
+CLOUDSQL_HOST=your_cloudsql_host
+CLOUDSQL_DB=your_database_name
+CLOUDSQL_PORT=5432
+CLOUDSQL_SSL_MODE=prefer
+CLOUDSQL_SCHEMA=public  # Source schema to export from
+
+# Supabase Configuration
+SUPABASE_USER=postgres
+SUPABASE_HOST=your_supabase_host.supabase.co
+SUPABASE_DB=postgres
+SUPABASE_PASSWORD=your_supabase_password
+SUPABASE_PORT=5432
+SUPABASE_SSL_MODE=require
+SUPABASE_SCHEMA=public  # Target schema to import into
+
+# Output Configuration
+OUTPUT_DIR=./outputs
+OUTPUT_DUMP=backup.sql
+CLEANED_DUMP=cleaned_backup.sql
+```
 
 ## Usage
+
+### Validate Your Configuration
+
+Before running any migration, check that your configuration is valid:
+
+```bash
+python main.py validate
+```
+
+### Full Migration
+
+Run a complete migration from CloudSQL to Supabase in one command:
+
 ```bash
 python main.py migrate
-python main.py backup
 ```
 
-## Setup
+You'll be prompted for your CloudSQL password during the process.
+
+### Full Migration with Custom Schema
+
+To migrate data from a specific CloudSQL schema to a custom Supabase schema:
+
+```bash
+python main.py migrate --source-schema=my_source_schema --target-schema=my_target_schema
+```
+
+### Using with a Manually Downloaded Dump File
+
+If you've already downloaded a PostgreSQL dump from CloudSQL:
+
 ```bash
-pip install -r requirements.txt
-cp .env.example .env
+# First clean the dump for Supabase compatibility
+python main.py clean-dump --input-file your_dump.sql --target-schema=my_schema
+
+# Then import the cleaned dump to Supabase
+python main.py import-db --input-file cleaned_backup.sql --schema=my_schema
 ```
+
+### Step-by-Step Migration
+
+You can also run each step of the migration separately:
+
+1. Export from CloudSQL:
+   ```bash
+   python main.py backup
+
+## Requirements
+
+- Python 3.8+
+- PostgreSQL client tools (psql, pg_dump)
+- Access to both CloudSQL and Supabase databases
+
+## Troubleshooting
+
+### Common Issues
+
+1. **"Command not found" errors**
+   - Ensure that PostgreSQL client tools are installed and in your PATH
+   - On Ubuntu/Debian: `sudo apt-get install postgresql-client`
+   - On macOS with Homebrew: `brew install libpq` and `brew link --force libpq`
+
+2. **Connection errors**
+   - Verify your database credentials in the `.env` file
+   - Check that your IP is allowed in both CloudSQL and Supabase network settings
+   - Test connection manually: `psql -h your_host -U your_user -d your_db`
+
+3. **Import errors**
+   - Common Supabase import issues are fixed by the cleaning step
+   - For specific table errors, you may need to customize the `clean.py` replacement rules
+
+### Getting Help
+
+For more detailed logs, use the `--verbose` flag with any command:
+
+```bash
+python main.py migrate --verbose
+```

cloudsql_to_supabase/cli.py

@@ -18,20 +18,26 @@ def cli(verbose):
 @click.option('--schema-only', is_flag=True, help="Only export database schema, not data")
 @click.option('--skip-export', is_flag=True, help="Skip the export step (use existing dump)")
 @click.option('--skip-clean', is_flag=True, help="Skip the cleaning step")
-def migrate(cloudsql_password, schema_only, skip_export, skip_clean):
+@click.option('--source-schema', help="Source schema in CloudSQL to export")
+@click.option('--target-schema', help="Target schema in Supabase to import into")
+def migrate(cloudsql_password, schema_only, skip_export, skip_clean, source_schema, target_schema):
     """Run full migration from CloudSQL to Supabase"""
     try:
         if not skip_export:
-            export.export_cloudsql(password=cloudsql_password, schema_only=schema_only)
+            export.export_cloudsql(
+                password=cloudsql_password, 
+                schema_only=schema_only,
+                schema=source_schema
+            )
         else:
             logger.info("Skipping export step")
 
         if not skip_clean:
-            clean.clean_dump_file()
+            clean.clean_dump_file(target_schema=target_schema)
         else:
             logger.info("Skipping clean step")
 
-        import_.import_to_supabase()
+        import_.import_to_supabase(schema=target_schema)
         click.echo("Migration completed successfully!")
 
     except Exception as e:
@@ -42,10 +48,15 @@ def migrate(cloudsql_password, schema_only, skip_export, skip_clean):
 @cli.command()
 @click.option('--cloudsql-password', help="CloudSQL password (if not provided, will prompt)")
 @click.option('--schema-only', is_flag=True, help="Only export database schema, not data")
-def backup(cloudsql_password, schema_only):
+@click.option('--schema', help="Specific schema to export (default: public)")
+def backup(cloudsql_password, schema_only, schema):
     """Only export CloudSQL to a dump file"""
     try:
-        dump_file = export.export_cloudsql(password=cloudsql_password, schema_only=schema_only)
+        dump_file = export.export_cloudsql(
+            password=cloudsql_password, 
+            schema_only=schema_only,
+            schema=schema
+        )
         click.echo(f"Backup completed successfully! File saved to: {dump_file}")
     except Exception as e:
         logger.error(f"Backup failed: {str(e)}")
@@ -55,13 +66,14 @@ def backup(cloudsql_password, schema_only):
 @cli.command()
 @click.option('--input-file', '-i', type=click.Path(exists=True), help="Input SQL dump file")
 @click.option('--output-file', '-o', type=click.Path(), help="Output cleaned SQL file")
-def clean_dump(input_file, output_file):
+@click.option('--target-schema', help="Target schema in Supabase to import into")
+def clean_dump(input_file, output_file, target_schema):
     """Clean a SQL dump file for Supabase compatibility"""
     try:
         input_path = Path(input_file) if input_file else None
         output_path = Path(output_file) if output_file else None
 
-        result_file = clean.clean_dump_file(input_path, output_path)
+        result_file = clean.clean_dump_file(input_path, output_path, target_schema)
         click.echo(f"Cleaning completed successfully! File saved to: {result_file}")
     except Exception as e:
         logger.error(f"Cleaning failed: {str(e)}")
@@ -70,11 +82,12 @@ def clean_dump(input_file, output_file):
 
 @cli.command()
 @click.option('--input-file', '-i', type=click.Path(exists=True), help="Input SQL dump file to import")
-def import_db(input_file):
+@click.option('--schema', help="Target schema in Supabase to import into")
+def import_db(input_file, schema):
     """Import a cleaned SQL dump file into Supabase"""
     try:
         input_path = Path(input_file) if input_file else None
-        import_.import_to_supabase(input_path)
+        import_.import_to_supabase(input_path, schema=schema)
         click.echo("Import completed successfully!")
     except Exception as e:
         logger.error(f"Import failed: {str(e)}")
@@ -98,4 +111,4 @@ def validate():
 
     except Exception as e:
         click.echo(f"Configuration error: {str(e)}", err=True)
-        exit(1)
+        exit(1)