feat: add yugabyte migration scripts

scripts/sunbird-yugabyte-migrations/README.md

@@ -0,0 +1,90 @@
+# Sunbird YugabyteDB Migrations
+
+This repository contains CQL migration scripts for migrating Sunbird database schemas to YugabyteDB.
+
+## Structure
+
+```
+sunbird-lern/          # Learning and user management schemas
+sunbird-knowlg/        # Knowledge and content management schemas  
+sunbird-inquiry/       # Assessment and inquiry schemas
+```
+
+## Usage
+
+### Running as Kubernetes Job
+
+In your Kubernetes job container, use the following script:
+
+```bash
+#!/bin/bash
+
+# Set environment variables
+export ENV=dev  # or sb, prod
+export YCQLSH_HOST=localhost
+export YCQLSH_PORT=9042
+export YCQLSH_USERNAME=yugabyte
+export YCQLSH_PASSWORD=yugabyte
+
+# Navigate to the migration directory
+cd /path/to/sunbird-yugabyte-migrations
+
+# Run sunbird-lern migrations
+cd sunbird-lern
+./execute_migrations.sh
+
+# Run sunbird-knowlg migrations
+cd ../sunbird-knowlg
+./execute_migrations.sh $ENV
+
+# Run sunbird-inquiry migrations
+cd ../sunbird-inquiry
+./execute_migrations.sh $ENV
+```
+
+### Manual Execution
+
+**1. Copy files to YugabyteDB pod:**
+```bash
+kubectl cp sunbird-lern/ -n <namespace> <pod-name>:/tmp/sunbird-lern/
+kubectl cp sunbird-knowlg/ -n <namespace> <pod-name>:/tmp/sunbird-knowlg/
+kubectl cp sunbird-inquiry/ -n <namespace> <pod-name>:/tmp/sunbird-inquiry/
+```
+
+**2. Run migrations inside the pod:**
+```bash
+# sunbird-lern
+cd /tmp/sunbird-lern
+./execute_migrations.sh
+
+# sunbird-knowlg
+cd /tmp/sunbird-knowlg
+./execute_migrations.sh dev
+
+# sunbird-inquiry
+cd /tmp/sunbird-inquiry
+./execute_migrations.sh dev
+```
+
+## Environment Parameters
+
+- **sunbird-lern**: No environment parameter (uses fixed keyspace names)
+- **sunbird-knowlg**: Requires environment (dev/sb/prod) - creates `{ENV}_category_store`, `{ENV}_content_store`, etc.
+- **sunbird-inquiry**: Requires environment (dev/sb/prod) - creates `{ENV}_hierarchy_store`, `{ENV}_question_store`
+
+## Connection Configuration
+
+Override default connection settings using environment variables:
+```bash
+export YCQLSH_HOST=localhost
+export YCQLSH_PORT=9042
+export YCQLSH_USERNAME=yugabyte
+export YCQLSH_PASSWORD=yugabyte
+```
+
+## Output
+
+Each script generates:
+- Colored console output showing progress
+- Timestamped log file in the same directory
+- Summary report of successful/failed migrations

scripts/sunbird-yugabyte-migrations/sunbird-inquiry/execute_migrations.sh

@@ -0,0 +1,187 @@
+#!/bin/bash
+
+#####################################################
+# YugabyteDB CQL Migration Script
+# Executes all CQL files in the sunbird-inquiry folder
+#####################################################
+
+# Usage function
+usage() {
+    echo "Usage: $0 [ENVIRONMENT]"
+    echo "  ENVIRONMENT: Environment prefix for CQL files (e.g., dev, sb, prod)"
+    echo "               Default: dev"
+    echo ""
+    echo "Examples:"
+    echo "  $0           # Uses 'dev' as environment"
+    echo "  $0 dev       # Uses 'dev' as environment"
+    echo "  $0 sb        # Uses 'sb' as environment"
+    echo "  $0 prod      # Uses 'prod' as environment"
+    exit 1
+}
+
+# Get environment from parameter or use default
+ENVIRONMENT="${1:-dev}"
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Get the directory where the script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Log file
+LOG_FILE="${SCRIPT_DIR}/migration_$(date +%Y%m%d_%H%M%S).log"
+
+# YugabyteDB connection parameters (can be overridden by environment variables)
+YCQLSH_HOST="${YCQLSH_HOST:-localhost}"
+YCQLSH_PORT="${YCQLSH_PORT:-9042}"
+YCQLSH_USERNAME="${YCQLSH_USERNAME:-yugabyte}"
+YCQLSH_PASSWORD="${YCQLSH_PASSWORD:-yugabyte}"
+
+# Counter variables
+TOTAL_FILES=0
+SUCCESSFUL_FILES=0
+FAILED_FILES=0
+
+# Array to store failed files
+declare -a FAILED_FILE_LIST
+
+#####################################################
+# Function: Print colored message
+#####################################################
+print_message() {
+    local color=$1
+    local message=$2
+    echo -e "${color}${message}${NC}"
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${message}" >> "${LOG_FILE}"
+}
+
+#####################################################
+# Function: Print header
+#####################################################
+print_header() {
+    echo ""
+    print_message "${BLUE}" "=============================================="
+    print_message "${BLUE}" "$1"
+    print_message "${BLUE}" "=============================================="
+    echo ""
+}
+
+#####################################################
+# Function: Execute CQL file
+#####################################################
+execute_cql_file() {
+    local cql_file=$1
+    local filename=$(basename "${cql_file}")
+
+    print_message "${YELLOW}" "Processing: ${filename}"
+
+    # Create temp file and replace ${ENV} with actual environment
+    local temp_file="${SCRIPT_DIR}/.tmp_${filename}"
+    sed "s/\${ENV}/${ENVIRONMENT}/g" "${cql_file}" > "${temp_file}"
+
+    # Execute the CQL file using ycqlsh
+    set +e  # Disable exit on error for this command
+    ycqlsh "${YCQLSH_HOST}" "${YCQLSH_PORT}" \
+        -u "${YCQLSH_USERNAME}" \
+        -p "${YCQLSH_PASSWORD}" \
+        -f "${temp_file}" >> "${LOG_FILE}" 2>&1
+    local exit_code=$?
+    set -e  # Re-enable exit on error
+
+    # Clean up temp file
+    rm -f "${temp_file}"
+
+    if [ $exit_code -eq 0 ]; then
+        print_message "${GREEN}" "✓ SUCCESS: ${filename} executed successfully"
+        SUCCESSFUL_FILES=$((SUCCESSFUL_FILES + 1))
+    else
+        print_message "${RED}" "✗ FAILED: ${filename} execution failed (exit code: $exit_code)"
+        FAILED_FILE_LIST+=("${filename}")
+        FAILED_FILES=$((FAILED_FILES + 1))
+    fi
+
+    echo ""
+}
+
+#####################################################
+# Main Script
+#####################################################
+
+print_header "YugabyteDB CQL Migration Script - sunbird-inquiry"
+
+print_message "${BLUE}" "Configuration:"
+echo "  Environment: ${ENVIRONMENT}"
+echo "  Host: ${YCQLSH_HOST}"
+echo "  Port: ${YCQLSH_PORT}"
+echo "  Username: ${YCQLSH_USERNAME}"
+echo "  Script Directory: ${SCRIPT_DIR}"
+echo "  Log File: ${LOG_FILE}"
+echo ""
+
+# Check if ycqlsh is available
+if ! command -v ycqlsh &> /dev/null; then
+    print_message "${RED}" "ERROR: ycqlsh command not found. Please ensure YugabyteDB client is installed."
+    exit 1
+fi
+
+# Test connection to YugabyteDB
+print_message "${YELLOW}" "Testing connection to YugabyteDB..."
+if ycqlsh "${YCQLSH_HOST}" "${YCQLSH_PORT}" \
+    -u "${YCQLSH_USERNAME}" \
+    -p "${YCQLSH_PASSWORD}" \
+    -e "DESCRIBE KEYSPACES;" >> "${LOG_FILE}" 2>&1; then
+    print_message "${GREEN}" "✓ Connection successful"
+    echo ""
+else
+    print_message "${RED}" "✗ Connection failed. Please check your connection parameters."
+    exit 1
+fi
+
+# Define the order of execution for CQL files
+CQL_FILES=(
+    "hierarchy_store.cql"
+    "question_store.cql"
+)
+
+print_header "Starting CQL File Execution"
+
+# Execute each CQL file in order
+for cql_file in "${CQL_FILES[@]}"; do
+    full_path="${SCRIPT_DIR}/${cql_file}"
+
+    if [ -f "${full_path}" ]; then
+        TOTAL_FILES=$((TOTAL_FILES + 1))
+        execute_cql_file "${full_path}"
+    else
+        print_message "${YELLOW}" "WARNING: ${cql_file} not found, skipping..."
+        echo ""
+    fi
+done
+
+# Print summary
+print_header "Migration Summary"
+
+echo "Total Files Processed: ${TOTAL_FILES}"
+echo "Successful: ${SUCCESSFUL_FILES}"
+echo "Failed: ${FAILED_FILES}"
+echo ""
+
+if [ ${FAILED_FILES} -gt 0 ]; then
+    print_message "${RED}" "Failed Files:"
+    for failed_file in "${FAILED_FILE_LIST[@]}"; do
+        echo "  - ${failed_file}"
+    done
+    echo ""
+    print_message "${RED}" "Migration completed with errors. Check log file: ${LOG_FILE}"
+    exit 1
+else
+    print_message "${GREEN}" "All migrations completed successfully!"
+    print_message "${BLUE}" "Log file: ${LOG_FILE}"
+fi
+
+echo ""
+print_message "${BLUE}" "=============================================="

scripts/sunbird-yugabyte-migrations/sunbird-inquiry/hierarchy_store.cql

@@ -0,0 +1,12 @@
+CREATE KEYSPACE IF NOT EXISTS ${ENV}_hierarchy_store WITH replication = {
+  'class': 'SimpleStrategy',
+  'replication_factor': '1'
+};
+
+CREATE TABLE IF NOT EXISTS ${ENV}_hierarchy_store.questionset_hierarchy (
+  identifier text,
+  hierarchy text,
+  instructions text,
+  outcomeDeclaration text,
+  PRIMARY KEY (identifier)
+);

scripts/sunbird-yugabyte-migrations/sunbird-inquiry/question_store.cql

@@ -0,0 +1,20 @@
+CREATE KEYSPACE IF NOT EXISTS ${ENV}_question_store WITH replication = {
+  'class': 'SimpleStrategy',
+  'replication_factor': '1'
+};
+
+CREATE TABLE IF NOT EXISTS ${ENV}_question_store.question_data (
+  identifier text,
+  body blob,
+  editorState text,
+  answer blob,
+  solutions text,
+  instructions text,
+  hints text,
+  media text,
+  responseDeclaration text,
+  interactions text,
+  outcomeDeclaration text,
+  feedback text,
+  PRIMARY KEY (identifier)
+);

scripts/sunbird-yugabyte-migrations/sunbird-knowlg/category_store.cql

@@ -0,0 +1,21 @@
+CREATE KEYSPACE IF NOT EXISTS ${ENV}_category_store WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
+
+CREATE TABLE IF NOT EXISTS ${ENV}_category_store.category_definition_data (
+    identifier text PRIMARY KEY,
+    forms map<text, text>,
+    objectmetadata map<text, text>
+) WITH bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE'
+    AND transactions = {'enabled': 'true'};

scripts/sunbird-yugabyte-migrations/sunbird-knowlg/content_store.cql

@@ -0,0 +1,50 @@
+CREATE KEYSPACE IF NOT EXISTS ${ENV}_content_store WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
+
+
+CREATE TABLE IF NOT EXISTS ${ENV}_content_store.question_data (
+    question_id text PRIMARY KEY,
+    body blob,
+    editorstate blob,
+    last_updated_on timestamp,
+    question blob,
+    solutions blob
+) WITH bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE'
+    AND transactions = {'enabled': 'true'};
+
+
+CREATE TABLE IF NOT EXISTS ${ENV}_content_store.content_data (
+    content_id text PRIMARY KEY,
+    body blob,
+    externallink text,
+    last_updated_on timestamp,
+    oldbody blob,
+    screenshots blob,
+    stageicons blob
+) WITH bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE'
+    AND transactions = {'enabled': 'true'};