From 95519bd5eefa9e053ae96fd5493c501791c8f2f4 Mon Sep 17 00:00:00 2001 From: zchpeter Date: Tue, 4 Nov 2025 12:46:02 -0500 Subject: [PATCH] feat: doc update for gitops and SDL --- mintlify/changelog/bytebase-1-8-0.mdx | 2 +- mintlify/docs.json | 44 ++- .../best-practices/file-organization.mdx | 185 +++++++++++ .../gitops/best-practices/git-and-cicd.mdx | 231 +++++++++++++ .../best-practices/migration-guidelines.mdx | 311 ++++++++++++++++++ mintlify/gitops/best-practices/overview.mdx | 82 +++++ .../best-practices/performance-and-drift.mdx | 299 +++++++++++++++++ .../sql-review-and-security.mdx | 251 ++++++++++++++ mintlify/gitops/develop.mdx | 42 --- .../migration-based-workflow/develop.mdx | 189 +++++++++++ .../migration-based-workflow/limitations.mdx | 82 +++++ .../migration-based-workflow/overview.mdx | 78 +++++ .../migration-based-workflow/release.mdx | 193 +++++++++++ .../sql-review-ci.mdx | 165 ++++++++++ mintlify/gitops/overview.mdx | 153 ++++++++- mintlify/gitops/release.mdx | 38 --- mintlify/gitops/sql-review-ci.mdx | 42 --- .../gitops/state-based-workflow/develop.mdx | 230 +++++++++++++ .../state-based-workflow/limitations.mdx | 166 ++++++++++ .../gitops/state-based-workflow/overview.mdx | 90 +++++ .../gitops/state-based-workflow/release.mdx | 145 ++++++++ .../state-based-workflow/sql-review-ci.mdx | 165 ++++++++++ .../gitops/troubleshooting/cicd-issues.mdx | 290 ++++++++++++++++ mintlify/gitops/troubleshooting/overview.mdx | 96 ++++++ .../troubleshooting/release-and-plan.mdx | 228 +++++++++++++ mintlify/gitops/troubleshooting/rollout.mdx | 274 +++++++++++++++ .../gitops/troubleshooting/sdl-issues.mdx | 230 +++++++++++++ 27 files changed, 4159 insertions(+), 142 deletions(-) create mode 100644 mintlify/gitops/best-practices/file-organization.mdx create mode 100644 mintlify/gitops/best-practices/git-and-cicd.mdx create mode 100644 mintlify/gitops/best-practices/migration-guidelines.mdx create mode 100644 mintlify/gitops/best-practices/overview.mdx create mode 100644 mintlify/gitops/best-practices/performance-and-drift.mdx create mode 100644 mintlify/gitops/best-practices/sql-review-and-security.mdx delete mode 100644 mintlify/gitops/develop.mdx create mode 100644 mintlify/gitops/migration-based-workflow/develop.mdx create mode 100644 mintlify/gitops/migration-based-workflow/limitations.mdx create mode 100644 mintlify/gitops/migration-based-workflow/overview.mdx create mode 100644 mintlify/gitops/migration-based-workflow/release.mdx create mode 100644 mintlify/gitops/migration-based-workflow/sql-review-ci.mdx delete mode 100644 mintlify/gitops/release.mdx delete mode 100644 mintlify/gitops/sql-review-ci.mdx create mode 100644 mintlify/gitops/state-based-workflow/develop.mdx create mode 100644 mintlify/gitops/state-based-workflow/limitations.mdx create mode 100644 mintlify/gitops/state-based-workflow/overview.mdx create mode 100644 mintlify/gitops/state-based-workflow/release.mdx create mode 100644 mintlify/gitops/state-based-workflow/sql-review-ci.mdx create mode 100644 mintlify/gitops/troubleshooting/cicd-issues.mdx create mode 100644 mintlify/gitops/troubleshooting/overview.mdx create mode 100644 mintlify/gitops/troubleshooting/release-and-plan.mdx create mode 100644 mintlify/gitops/troubleshooting/rollout.mdx create mode 100644 mintlify/gitops/troubleshooting/sdl-issues.mdx diff --git a/mintlify/changelog/bytebase-1-8-0.mdx b/mintlify/changelog/bytebase-1-8-0.mdx index a36c068a2..f9ff56aad 100644 --- a/mintlify/changelog/bytebase-1-8-0.mdx +++ b/mintlify/changelog/bytebase-1-8-0.mdx @@ -11,7 +11,7 @@ import InstallUpgrade from '/snippets/install/install-upgrade.mdx'; - Support syncing schema for MySQL. - Support approving issues via Lark (Feishu). -- Support [SQL Review CI in the GitOps workflow](/gitops/sql-review-ci). +- Support [SQL Review CI in the GitOps workflow](/gitops/migration-based-workflow/sql-review-ci). - DBAs and workspace admins can run SQL statements in Admin mode from the SQL Editor. - Support altering multiple databases on tenant mode. diff --git a/mintlify/docs.json b/mintlify/docs.json index 1ce6d7081..6587d7d2e 100644 --- a/mintlify/docs.json +++ b/mintlify/docs.json @@ -130,9 +130,47 @@ "pages": [ "gitops/overview", "gitops/installation", - "gitops/develop", - "gitops/sql-review-ci", - "gitops/release" + { + "group": "Migration-Based Workflow", + "pages": [ + "gitops/migration-based-workflow/overview", + "gitops/migration-based-workflow/develop", + "gitops/migration-based-workflow/sql-review-ci", + "gitops/migration-based-workflow/release", + "gitops/migration-based-workflow/limitations" + ] + }, + { + "group": "State-Based Workflow (SDL)", + "pages": [ + "gitops/state-based-workflow/overview", + "gitops/state-based-workflow/develop", + "gitops/state-based-workflow/sql-review-ci", + "gitops/state-based-workflow/release", + "gitops/state-based-workflow/limitations" + ] + }, + { + "group": "Best Practices", + "pages": [ + "gitops/best-practices/overview", + "gitops/best-practices/file-organization", + "gitops/best-practices/migration-guidelines", + "gitops/best-practices/git-and-cicd", + "gitops/best-practices/sql-review-and-security", + "gitops/best-practices/performance-and-drift" + ] + }, + { + "group": "Troubleshooting", + "pages": [ + "gitops/troubleshooting/overview", + "gitops/troubleshooting/release-and-plan", + "gitops/troubleshooting/rollout", + "gitops/troubleshooting/sdl-issues", + "gitops/troubleshooting/cicd-issues" + ] + } ] }, { diff --git a/mintlify/gitops/best-practices/file-organization.mdx b/mintlify/gitops/best-practices/file-organization.mdx new file mode 100644 index 000000000..3d83a7389 --- /dev/null +++ b/mintlify/gitops/best-practices/file-organization.mdx @@ -0,0 +1,185 @@ +--- +title: File Organization +--- + +Proper file organization is crucial for maintainable GitOps workflows. This guide covers repository structures and directory layouts that scale with your team. + +## Repository Structure + +### Monorepo Approach + +Database migrations alongside application code: + +``` +my-app/ +├── src/ # Application code +├── migrations/ # Database migrations +│ ├── versioned/ +│ │ ├── 001__init.sql +│ │ └── 002__add_users.sql +│ └── schema/ +│ └── public.sql # SDL if using declarative +├── .github/workflows/ +│ └── bytebase-gitops.yml # CI/CD integration +└── README.md +``` + +**Benefits:** +- Migrations versioned with application code +- Atomic commits for schema + code changes +- Single source of truth + +**Best for:** +- Small to medium teams +- Tight schema-code coupling +- Monolithic or modular monolith architectures + +### Separate Repository + +Dedicated database repository: + +``` +database-schemas/ +├── app-db/ +│ ├── migrations/ +│ └── schema/ +├── analytics-db/ +│ └── migrations/ +└── .gitlab-ci.yml +``` + +**Benefits:** +- Separation of concerns +- Independent deployment cycles +- Multiple teams/databases + +**Best for:** +- Larger organizations +- Dedicated database teams +- Microservices with shared databases + + + Choose based on your team structure. Co-located migrations work well for small teams with tight schema-code coupling. Separate repos fit larger organizations with dedicated database teams. + + +## Directory Layout + +### For Migration-Based Workflow + +**Option 1: Organized by Category** + +``` +migrations/ +├── baseline/ +│ └── 000__initial_schema.sql # Initial baseline +├── features/ +│ ├── 001__users.sql +│ ├── 002__products.sql +│ └── 003__orders.sql +└── hotfixes/ + └── 004__fix_index.sql +``` + +**Benefits:** +- Clear organization by purpose +- Easy to navigate +- Separates routine changes from emergencies + +**Option 2: Flat Structure** + +``` +migrations/ +├── 001__initial_schema.sql +├── 002__add_users.sql +├── 003__add_products_dml.sql +└── 004__add_indexes.sql +``` + +**Benefits:** +- Simple and straightforward +- Chronological ordering +- Easy to understand + +### For State-Based Workflow (SDL) + +**Option 1: Organized by Schema** + +``` +schema/ +├── public.sql # Main schema +├── analytics.sql # Analytics schema +└── internal.sql # Internal schema +``` + +**Benefits:** +- Clear schema separation +- Matches database structure +- Easy to find objects + +**Option 2: Split by Object Type** + +``` +schema/ +├── 01_tables.sql +├── 02_indexes.sql +├── 03_views.sql +├── 04_functions.sql +└── 05_sequences.sql +``` + +**Benefits:** +- Organized by DDL type +- Predictable file structure +- Clear dependencies (tables before indexes) + +### Hybrid Approach + +Combine both workflows for different purposes: + +``` +database/ +├── schema/ # SDL for structure (DDL) +│ ├── public.sql +│ └── analytics.sql +├── migrations/ # Migrations for data (DML) +│ ├── 001__seed_roles_dml.sql +│ └── 002__migrate_users_dml.sql +└── .github/workflows/ + ├── schema-cicd.yml # SDL pipeline + └── data-cicd.yml # Migration pipeline +``` + +**Best for:** +- Teams wanting declarative schema management +- Projects requiring data migrations +- Gradual migration from versioned to SDL + +## Documentation Structure + +Maintain supporting documentation alongside migrations: + +``` +database/ +├── migrations/ +├── schema/ +├── docs/ +│ ├── CHANGELOG.md # Schema changelog +│ ├── DEPENDENCIES.md # Schema-app dependencies +│ └── ROLLBACK_PLAN.md # Rollback procedures +└── test-data/ + └── seed.sql # Test data for development +``` + +--- + +## Next Steps + + + + Learn version numbering and file best practices + + + + Set up branching strategies and pipelines + + diff --git a/mintlify/gitops/best-practices/git-and-cicd.mdx b/mintlify/gitops/best-practices/git-and-cicd.mdx new file mode 100644 index 000000000..5fa69e00a --- /dev/null +++ b/mintlify/gitops/best-practices/git-and-cicd.mdx @@ -0,0 +1,231 @@ +--- +title: Git and CI/CD +--- + +Choose the right Git branching strategy and CI/CD patterns for your team's workflow. + +## Git Branching Strategies + +### GitHub Flow (Simple) + +``` +main ─────────────○─────────────○─────→ + ╲ ╱ ╱ + feature-branch hotfix-branch +``` + +**Workflow:** +1. Create feature branch from `main` +2. Develop migration +3. Open PR, SQL review runs +4. Merge to `main` +5. CI creates release and deploys + +**Best for:** Continuous deployment, small teams + +### GitLab Flow (Environment Branches) + +``` +main ─────────○─────────○─────────────→ + ╲ ╲ + staging production +``` + +**Workflow:** +1. Merge to `main` → deploys to dev +2. Merge `main` to `staging` → deploys to staging +3. Merge `staging` to `production` → deploys to prod + +**Best for:** Progressive environment promotion + +### Git Flow (Release Branches) + +``` +main ──────────────────○────────○────→ (production) + ╲ ╱ ╱ + develop ─○─○─────○─── (dev) + ╲ ╱ + feature-branch +``` + +**Workflow:** +1. Develop in `feature` branches +2. Merge to `develop` → deploys to dev/staging +3. Create `release` branch → final testing +4. Merge to `main` → deploys to production + +**Best for:** Scheduled releases, large teams + +## CI/CD Integration Patterns + +### Pattern 1: Automated Review + Manual Deploy + +```yaml +# .github/workflows/sql-review.yml +on: pull_request + +jobs: + sql-review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: SQL Review + run: | + # Call Bytebase API to check release + curl -X POST "$BB_URL/v1/projects/my-project/releases:check" +``` + +```yaml +# .github/workflows/deploy.yml +on: + workflow_dispatch: # Manual trigger + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Create Release + run: | + # Create release via API +``` + +**Best for:** Production environments requiring manual control + +### Pattern 2: Fully Automated Pipeline + +```yaml +# .github/workflows/database-cicd.yml +on: + push: + branches: [main] + paths: ['migrations/**'] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create Release + id: release + run: | + RELEASE=$(curl -X POST "$BB_URL/v1/projects/my-project/releases" ...) + echo "release=$RELEASE" >> $GITHUB_OUTPUT + + - name: Create Plan + id: plan + run: | + PLAN=$(curl -X POST "$BB_URL/v1/projects/my-project/plans" \ + -d '{"release": "${{ steps.release.outputs.release }}"}') + echo "plan=$PLAN" >> $GITHUB_OUTPUT + + - name: Create Rollout + run: | + curl -X POST "$BB_URL/v1/projects/my-project/rollouts" \ + -d '{"plan": "${{ steps.plan.outputs.plan }}"}' +``` + +**Best for:** Dev/staging environments, high-frequency deployments + + + Complete CI/CD setup examples for GitHub, GitLab, Azure DevOps, and Bitbucket + + +## Testing Strategies + +### Test in Non-Production First + +Always follow environment progression: + +``` +Local Dev → CI Testing → Dev Environment → Staging → Production +``` + +**Never skip environments for production changes.** + +### Use Sampling for Large Fleets + +For database groups with 100+ databases: + +```json +{ + "project": { + "ci_sampling_size": 20 + } +} +``` + +During SQL review, validates on 20 random databases for faster feedback. + +### Maintain Test Data + +Create sample datasets that mirror production: + +```sql +-- test-data/seed.sql +-- Realistic test data for development +INSERT INTO users (username, email) VALUES + ('alice', 'alice@example.com'), + ('bob', 'bob@example.com'); + +INSERT INTO orders (user_id, total) VALUES + (1, 99.99), + (2, 149.99); +``` + +Test migrations against this data locally before committing. + +### Schema Compatibility Testing + +Before deploying schema changes: + +1. **Deploy new schema to staging** +2. **Test old application version against new schema** +3. **Verify backward compatibility** +4. **Deploy application update** +5. **Remove deprecated columns/tables later** + +This enables zero-downtime deployments. + +### Local Testing Workflow + +**Before committing:** + +```bash +# 1. Apply migration locally +psql $LOCAL_DB -f migrations/new_migration.sql + +# 2. Test with sample data +psql $LOCAL_DB -f test-data/seed.sql + +# 3. Run application tests +npm test + +# 4. Verify rollback (if applicable) +psql $LOCAL_DB -f migrations/rollback.sql +``` + +### CI Testing Checklist + +Automated tests should verify: + +- ✅ SQL syntax is valid +- ✅ Migration applies successfully +- ✅ No SQL review rule violations +- ✅ Schema changes don't break existing queries +- ✅ Performance impact is acceptable +- ✅ Rollback script works (if provided) + +--- + +## Next Steps + + + + Configure review rules and security practices + + + + Step-by-step CI/CD setup guides + + diff --git a/mintlify/gitops/best-practices/migration-guidelines.mdx b/mintlify/gitops/best-practices/migration-guidelines.mdx new file mode 100644 index 000000000..641daf6b0 --- /dev/null +++ b/mintlify/gitops/best-practices/migration-guidelines.mdx @@ -0,0 +1,311 @@ +--- +title: Migration Guidelines +--- + +Follow these guidelines to write maintainable, safe, and effective database migrations. + +## Version Numbering Strategies + +### Strategy 1: Timestamp-Based (Recommended) + +``` +20250120143000__add_user_email.sql +20250121091500__create_orders_table.sql +``` + +**Format:** `YYYYMMDDHHmmss` + +**Advantages:** +- ✅ No merge conflicts with parallel development +- ✅ Chronological ordering +- ✅ Supports distributed teams + +**Disadvantages:** +- ⚠️ Less human-readable +- ⚠️ Doesn't convey significance + +**Best for:** Teams with frequent parallel development + +### Strategy 2: Semantic Versioning + +``` +v1.0.0__initial_release.sql +v1.1.0__add_user_profiles.sql +v1.1.1__fix_profile_constraint.sql +v2.0.0__redesign_authentication.sql +``` + +**Format:** `v..` + +**Advantages:** +- ✅ Conveys change significance +- ✅ Aligns with application versioning +- ✅ Clear breaking change indication + +**Disadvantages:** +- ⚠️ Requires coordination +- ⚠️ Merge conflicts possible + +**Best for:** Teams with coordinated releases + +### Strategy 3: Sequential with Milestones + +``` +001__initial_schema.sql +002__add_users.sql +... +100__v1_0_release.sql +101__add_analytics.sql +... +200__v2_0_release.sql +``` + +**Advantages:** +- ✅ Simple and clear +- ✅ Milestones mark releases + +**Disadvantages:** +- ⚠️ Merge conflicts in parallel work +- ⚠️ Gaps can be confusing + +**Best for:** Small teams, sequential development + +## Migration File Best Practices + +### Keep Migrations Small + + + + ```sql + -- 005__add_user_email.sql + ALTER TABLE users ADD COLUMN email TEXT; + ``` + + - Single, clear purpose + - Fast execution + - Easy to review + - Simple to rollback + + + + ```sql + -- 005__big_refactor.sql + ALTER TABLE users ADD COLUMN email TEXT; + ALTER TABLE users ADD COLUMN phone TEXT; + CREATE TABLE user_preferences (...); + CREATE TABLE user_sessions (...); + INSERT INTO user_preferences ...; + -- 500 more lines + ``` + + - Multiple unrelated changes + - Long execution time + - Hard to review + - Difficult to troubleshoot + + + +**Guideline:** One logical change per file + +### Include Rollback Planning + +Document rollback approach in comments: + +```sql +-- 010__add_payment_methods_table.sql +-- Rollback: DROP TABLE payment_methods; +-- Impact: Requires app deployment v2.5.0+ + +CREATE TABLE payment_methods ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + card_last_four TEXT NOT NULL, + created_at TIMESTAMP DEFAULT NOW() +); +``` + +### Transaction Handling + + + **Bytebase Automatic Transactions**: Bytebase automatically wraps all SQL statements in a migration file within a single transaction. You typically don't need to add explicit `BEGIN`/`COMMIT` statements. + + +**When Bytebase's automatic transactions are sufficient:** +- Single-file migrations with multiple statements +- Standard DDL operations (CREATE, ALTER, DROP) +- Simple DML operations (INSERT, UPDATE, DELETE) + +**When you might need explicit transaction control:** +- **Batched operations** requiring commits between chunks +- **Long-running data migrations** that need progress checkpoints +- **Database-specific requirements** like PostgreSQL's `CREATE INDEX CONCURRENTLY` (which cannot run in a transaction) + +**Example of batched migration (when needed):** +```sql +-- 015__batch_archive_logs_dml.sql +-- Note: This uses explicit batching for large data operation +DO $$ +DECLARE + batch_size INT := 10000; + deleted_count INT; +BEGIN + LOOP + DELETE FROM logs + WHERE id IN ( + SELECT id FROM logs + WHERE created_at < '2024-01-01' + LIMIT batch_size + ); + + GET DIAGNOSTICS deleted_count = ROW_COUNT; + EXIT WHEN deleted_count < batch_size; + + COMMIT; -- Commit each batch + PERFORM pg_sleep(0.1); -- Throttle + END LOOP; +END $$; +``` + +**Database-specific transaction support:** +- ✅ PostgreSQL: DDL in transactions (except `CONCURRENTLY` operations) +- ❌ MySQL: DDL commits immediately (implicit commit) +- ✅ SQL Server: DDL in transactions + +### Add Comments for Complex Logic + +```sql +-- 020__migrate_legacy_permissions_dml.sql +-- Migrates old role system to new permission model +-- Old: roles.name -> New: permissions.scope + permissions.action +-- Mapping: +-- 'admin' -> 'database:*' +-- 'editor' -> 'database:write' +-- 'viewer' -> 'database:read' + +UPDATE permissions +SET + scope = CASE + WHEN roles.name = 'admin' THEN 'database' + WHEN roles.name = 'editor' THEN 'database' + WHEN roles.name = 'viewer' THEN 'database' + END, + action = CASE + WHEN roles.name = 'admin' THEN '*' + WHEN roles.name = 'editor' THEN 'write' + WHEN roles.name = 'viewer' THEN 'read' + END +FROM roles +WHERE permissions.role_id = roles.id; +``` + +## Documentation + +### Maintain Migration Changelog + +Document significant schema changes: + +```markdown +# Database Changelog + +## v2.0.0 (2025-01-20) +- Added `user_preferences` table for customization +- Migrated legacy role system to new permissions model +- **Breaking**: Removed deprecated `user_settings` table + +## v1.5.0 (2025-01-15) +- Added email column to users table +- Created indexes on frequently queried columns +``` + +### Document Schema Dependencies + +Track dependencies between application and schema: + +```markdown +# Schema Dependencies + +## users.email Column +- Added: v1.5.0 (migration 010) +- Required by: Auth Service v2.0+ +- Can remove after: All instances upgraded to v2.1+ +``` + +This helps coordinate schema cleanup with application deployments. + +## Common Anti-Patterns to Avoid + +### ❌ Modifying Applied Migrations + +**Don't:** +```sql +-- 005__add_email.sql (already deployed) +ALTER TABLE users ADD COLUMN email TEXT; -- Changed to VARCHAR +``` + +**Do:** +```sql +-- 006__fix_email_type.sql +ALTER TABLE users ALTER COLUMN email TYPE VARCHAR(255); +``` + +### ❌ Skipping Version Numbers + +**Don't:** +``` +001__init.sql +002__add_users.sql +010__add_products.sql ← Why skip 003-009? +``` + +**Do:** +``` +001__init.sql +002__add_users.sql +003__add_products.sql +``` + +### ❌ Mixing DDL and DML Without Suffix + +**Don't:** +```sql +-- 015__add_orders.sql +CREATE TABLE orders (...); +INSERT INTO orders VALUES (...); -- DML without _dml suffix +``` + +**Do:** +```sql +-- 015__add_orders.sql (DDL only) +CREATE TABLE orders (...); + +-- 016__seed_orders_dml.sql (DML separate) +INSERT INTO orders VALUES (...); +``` + +### ❌ Long-Running Migrations in Production + +**Don't:** +```sql +-- 020__huge_migration.sql +UPDATE users SET legacy_field = NULL; -- Locks 10M rows for 10 minutes +``` + +**Do:** +```sql +-- 020__migrate_users_batch_dml.sql +-- Batch processing with throttling (see Performance section) +``` + +--- + +## Next Steps + + + + Learn branching strategies and testing patterns + + + + Optimize migrations and handle drift + + diff --git a/mintlify/gitops/best-practices/overview.mdx b/mintlify/gitops/best-practices/overview.mdx new file mode 100644 index 000000000..6323cdffa --- /dev/null +++ b/mintlify/gitops/best-practices/overview.mdx @@ -0,0 +1,82 @@ +--- +title: Overview +--- + +This guide provides battle-tested patterns and recommendations for production GitOps workflows with Bytebase. + +## What You'll Learn + +This best practices guide covers: + +- **File Organization** - Repository structures and directory layouts for different team sizes +- **Migration Guidelines** - Version numbering strategies, file best practices, and common pitfalls +- **Git and CI/CD** - Branching strategies, CI/CD patterns, and testing approaches +- **SQL Review and Security** - Review configuration and security best practices +- **Performance and Drift** - Optimization techniques and drift management + +## Who Should Read This + +This guide is designed for: + +- **DevOps Engineers** setting up database CI/CD pipelines +- **Database Administrators** migrating to GitOps workflows +- **Development Teams** implementing database version control +- **Platform Engineers** building self-service database platforms + +## Prerequisites + +Before diving into best practices, you should have: + +- Basic understanding of [GitOps Workflow](/gitops/overview) +- Familiarity with either [Migration-Based](/gitops/migration-based-workflow/overview) or [State-Based](/gitops/state-based-workflow/overview) workflows +- [GitOps integration installed](/gitops/installation) + +## Quick Reference + +### Critical Best Practices + +**File Management:** +- Use timestamp-based versioning for parallel development +- Keep migrations small and focused (one logical change per file) +- Never modify applied migrations + +**Testing:** +- Always test in non-production first: Local → Dev → Staging → Production +- Maintain test data that mirrors production +- Test schema compatibility before deployment + +**Security:** +- Use service accounts for CI/CD +- Grant least privilege database access +- Store secrets in secret management systems + +**Performance:** +- Batch large data migrations +- Use online schema changes for large tables +- Create indexes concurrently when possible + +--- + +## Get Started + + + + Repository structures and directory layouts + + + + Version numbering and migration best practices + + + + Branching strategies and testing patterns + + + + Review configuration and security practices + + + + Optimization and drift management + + diff --git a/mintlify/gitops/best-practices/performance-and-drift.mdx b/mintlify/gitops/best-practices/performance-and-drift.mdx new file mode 100644 index 000000000..541c2e9d1 --- /dev/null +++ b/mintlify/gitops/best-practices/performance-and-drift.mdx @@ -0,0 +1,299 @@ +--- +title: Performance and Drift +--- + +Optimize migration performance and effectively manage schema drift in your GitOps workflow. + +## Performance Optimization + +### Batch Large Data Migrations + +For DML on large tables, process in batches: + +```sql +-- 050__archive_old_logs_dml.sql +-- Instead of single large DELETE: +-- ❌ DELETE FROM logs WHERE created_at < '2024-01-01'; + +-- Use batched approach: +DO $$ +DECLARE + batch_size INT := 10000; + deleted_count INT; +BEGIN + LOOP + DELETE FROM logs + WHERE id IN ( + SELECT id FROM logs + WHERE created_at < '2024-01-01' + LIMIT batch_size + ); + + GET DIAGNOSTICS deleted_count = ROW_COUNT; + EXIT WHEN deleted_count < batch_size; + + COMMIT; -- If supported + PERFORM pg_sleep(0.1); -- Throttle + END LOOP; +END $$; +``` + +**Benefits:** +- Reduces lock contention +- Prevents transaction log bloat +- Allows monitoring progress +- Can be paused/resumed + +### Use Online Schema Changes for Large Tables + +For MySQL tables > 1M rows: + +```sql +-- Instead of: +-- ❌ 010__add_index.sql +CREATE INDEX idx_users_email ON users(email); + +-- Use gh-ost: +-- ✅ 010__add_index_ghost.sql +ALTER TABLE users ADD INDEX idx_users_email (email); +``` + + + Configure gh-ost for zero-downtime MySQL migrations + + +### Optimize Index Creation + +Create indexes concurrently when possible: + +```sql +-- PostgreSQL: Non-blocking index creation +CREATE INDEX CONCURRENTLY idx_users_email ON users(email); + +-- MySQL: Use online DDL +ALTER TABLE users ADD INDEX idx_email (email), ALGORITHM=INPLACE, LOCK=NONE; +``` + +**Note:** `CONCURRENTLY` requires: +- Cannot be in transaction block +- Takes longer than regular index creation +- May fail if concurrent writes conflict + +### Timing Considerations + +**Best times for migrations:** +- ✅ Low-traffic hours (nights, weekends) +- ✅ During maintenance windows +- ✅ After application deployments + +**Avoid:** +- ❌ Peak business hours +- ❌ During critical operations (month-end, sales events) +- ❌ Without communication to stakeholders + +### Monitor Migration Performance + +Track execution time during development: + +```sql +-- Add timing to migration +\timing on +-- Migration SQL here +\timing off +``` + +Set expectations for production: +```sql +-- 015__large_migration.sql +-- Expected duration: 5-10 minutes +-- Locks: users table (write lock) +-- Tested on: 2M row table +``` + +## Handling Schema Drift + +Schema drift occurs when manual changes bypass GitOps. + +### Detection + +Bytebase drift detection identifies untracked changes: + + + Detect and manage schema drift + + +**How drift detection works:** +1. Bytebase periodically scans database schema +2. Compares with GitOps tracked schema +3. Identifies differences (new columns, indexes, etc.) +4. Alerts team of drift + +### Prevention + +**1. Restrict Direct Database Access** + +Use readonly users for most access: + +```sql +-- Default user: read-only +GRANT SELECT ON ALL TABLES IN SCHEMA public TO app_user; + +-- Special user for emergencies only +GRANT ALL ON DATABASE app_db TO emergency_user; +``` + +**2. Require All Changes Through GitOps** + +Enforce policy through: +- Database access controls +- Team documentation +- Code review requirements +- Regular audits + +**3. Monitor Drift Alerts** + +Set up notifications: + +```json +{ + "drift_detection": { + "enabled": true, + "schedule": "0 */6 * * *", // Every 6 hours + "notification": { + "channels": ["slack", "email"] + } + } +} +``` + +**4. Regular Drift Checks** + +Audit schema consistency: +- Weekly automated scans +- Monthly manual reviews +- After emergency changes + +### Resolution + +When drift is detected: + +**Option 1: Reverse the drift** +```sql +-- Revert manual change to match GitOps state +ALTER TABLE users DROP COLUMN unauthorized_column; +``` + +**Option 2: Incorporate into GitOps** +```sql +-- Create migration reflecting the change +-- 055__add_emergency_column.sql +ALTER TABLE users ADD COLUMN emergency_column TEXT; +``` + +**Decision matrix:** + +| Scenario | Action | +|----------|--------| +| Unauthorized change | Reverse immediately | +| Emergency hotfix | Document and create migration | +| Testing/development | Reverse in non-prod | +| Production emergency | Incorporate, review process | + +### Drift Incident Response + +When drift occurs: + +1. **Detect** - Alert received +2. **Assess** - Identify change and impact +3. **Communicate** - Notify team +4. **Resolve** - Reverse or incorporate +5. **Document** - Post-incident review +6. **Prevent** - Update processes + +**Incident template:** + +```markdown +# Drift Incident: [Date] + +## Summary +- **Detected:** 2025-01-20 14:30 UTC +- **Database:** production-db +- **Change:** Column `temp_fix` added to `users` table + +## Impact +- No customer impact +- Not tracked in GitOps + +## Root Cause +- Emergency hotfix bypassed GitOps +- Developer had elevated permissions + +## Resolution +- Created migration 056__add_temp_fix.sql +- Updated GitOps repository + +## Prevention +- Removed elevated permissions +- Added approval requirement for prod access +``` + +## Performance Monitoring + +### Track Migration Metrics + +Monitor these metrics: + +| Metric | Target | Alert Threshold | +|--------|--------|-----------------| +| Migration duration | < 5 min | > 10 min | +| Lock wait time | < 1 sec | > 5 sec | +| Rows affected | Documented | > Expected | +| Rollback success rate | 100% | < 100% | + +### Database Health Checks + +Before and after migrations: + +```sql +-- Check table sizes +SELECT + schemaname, + tablename, + pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS size +FROM pg_tables +WHERE schemaname = 'public' +ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC; + +-- Check index usage +SELECT + schemaname, + tablename, + indexname, + idx_scan, + pg_size_pretty(pg_relation_size(indexrelid)) AS size +FROM pg_stat_user_indexes +WHERE idx_scan < 100 +ORDER BY pg_relation_size(indexrelid) DESC; +``` + +--- + +## Next Steps + + + + Solutions for common GitOps issues + + + + Configure gh-ost for zero-downtime changes + + + + Set up automated drift detection + + + + Return to GitOps overview + + diff --git a/mintlify/gitops/best-practices/sql-review-and-security.mdx b/mintlify/gitops/best-practices/sql-review-and-security.mdx new file mode 100644 index 000000000..f02aeab4c --- /dev/null +++ b/mintlify/gitops/best-practices/sql-review-and-security.mdx @@ -0,0 +1,251 @@ +--- +title: SQL Review and Security +--- + +Configure SQL review rules and implement security best practices for production GitOps workflows. + +## SQL Review Configuration + +Configure SQL review rules to enforce standards across your team. + + + Configure 200+ linting rules for automated validation + + +### Recommended Rules + +**Critical Rules (ERROR level):** +- ❌ `DROP DATABASE` +- ❌ `DROP TABLE` (without confirmation) +- ❌ Missing `WHERE` clause in `UPDATE`/`DELETE` +- ❌ `NOT NULL` on existing columns without default +- ❌ Charset changes on existing columns + +**Warning Rules (WARN level):** +- ⚠️ Missing indexes on foreign keys +- ⚠️ Column without comments +- ⚠️ Table without primary key +- ⚠️ Large `IN` clause (> 1000 items) + +**Info Rules:** +- 💡 Consider partitioning for large tables +- 💡 Index naming convention suggestions + +### Example Policy + +```json +{ + "rule_list": [ + { + "type": "naming.table", + "level": "ERROR", + "payload": { + "format": "^[a-z_]+$" + } + }, + { + "type": "statement.select.no-select-all", + "level": "WARNING" + }, + { + "type": "column.required", + "level": "WARNING", + "payload": { + "column_list": ["created_at", "updated_at"] + } + } + ] +} +``` + +### Review Severity Levels + +Configure how different rule violations are handled: + +| Level | Behavior | Use Case | +|-------|----------|----------| +| **ERROR** | Blocks merge | Dangerous operations, critical standards | +| **WARNING** | Allows merge with approval | Best practices, style guidelines | +| **INFO** | Informational only | Suggestions, optimization tips | + +## Security Best Practices + +### Use Service Accounts + +Create dedicated service accounts for CI/CD: + +```bash +# Don't use personal accounts +❌ export BB_TOKEN="user-alice-token" + +# Use service accounts +✅ export BB_TOKEN="service-account-cicd-token" +``` + +**Service account setup:** +1. Create service account in Bytebase +2. Grant minimum required permissions +3. Store token in CI/CD secrets +4. Rotate tokens regularly + + + Learn about service account authentication + + +### Least Privilege Database Access + +Configure Bytebase with minimal database permissions: + +**For schema changes:** +```sql +GRANT CREATE, ALTER, DROP ON DATABASE app_db TO bytebase_user; +``` + +**For readonly access:** +```sql +GRANT SELECT ON ALL TABLES IN SCHEMA public TO bytebase_readonly; +``` + +Avoid using superuser accounts. + +### Protect Sensitive Migrations + +For migrations containing sensitive data: + +```sql +-- 099__seed_api_keys_dml.sql +-- WARNING: Contains sensitive data +-- Ensure this file is not committed to version control + +INSERT INTO api_credentials (service, key) VALUES + ('payment_gateway', '${PAYMENT_API_KEY}'), + ('email_service', '${EMAIL_API_KEY}'); +``` + +**Alternatives:** +- Store secrets in secret management systems (AWS Secrets Manager, HashiCorp Vault) +- Reference secrets via environment variables in CI/CD +- Use Bytebase secret integration + + + Configure database connections with secret managers + + +### Secrets Management + +**Option 1: CI/CD Secrets** + +```yaml +# .github/workflows/deploy.yml +env: + BYTEBASE_TOKEN: ${{ secrets.BYTEBASE_TOKEN }} + DB_PASSWORD: ${{ secrets.DB_PASSWORD }} +``` + +**Option 2: Secret Manager** + +```yaml +# Use AWS Secrets Manager +- name: Get secrets + run: | + aws secretsmanager get-secret-value \ + --secret-id bytebase/cicd \ + --query SecretString +``` + +**Option 3: External Secret Store** + +```bash +# Use HashiCorp Vault +export BYTEBASE_TOKEN=$(vault kv get -field=token secret/bytebase) +``` + +### Audit and Compliance + +Enable comprehensive audit logging: + +```json +{ + "project": { + "audit_log_retention_days": 365 + } +} +``` + +**What gets logged:** +- All schema changes +- Who approved changes +- When deployments occurred +- Access to sensitive data +- Policy violations + + + Configure audit logging for compliance + + +### Network Security + +**Restrict Bytebase Access:** +- Use VPN or private networking for production +- Enable IP allowlisting +- Use TLS for all connections +- Implement firewall rules + +**Database Connection Security:** +```yaml +# Use SSL/TLS for database connections +database: + ssl: + enabled: true + ca_cert: /path/to/ca.pem + verify_mode: require +``` + +### Role-Based Access Control + +Configure appropriate roles: + +| Role | Permissions | Use Case | +|------|-------------|----------| +| **Owner** | Full access | Team leads, admins | +| **DBA** | Schema changes, admin mode | Database administrators | +| **Developer** | Create issues, query data | Application developers | +| **Releaser** | Deploy to production | Release engineers | +| **Querier** | Query data only | Analysts, support | + + + Configure role-based access control + + +### Code Review Security + +Security checklist for PR/MR reviews: + +- ✅ No hardcoded secrets or passwords +- ✅ No `SELECT *` exposing sensitive columns +- ✅ Proper `WHERE` clauses to prevent mass updates +- ✅ No `DROP` statements without explicit approval +- ✅ Appropriate indexes to prevent performance issues +- ✅ Data access follows compliance requirements + +--- + +## Next Steps + + + + Optimize migrations and handle drift + + + + Complete reference of 200+ rules + + + + Configure audit logging + + + + Protect sensitive data + + diff --git a/mintlify/gitops/develop.mdx b/mintlify/gitops/develop.mdx deleted file mode 100644 index 3103588c3..000000000 --- a/mintlify/gitops/develop.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Develop ---- - -## Filename Requirements - -To ensure proper version control and execution, your SQL migration filenames must follow a specific structure. Each filename consists of three main parts: a **Version**, a **Description**, and an optional **Change Type Suffix**. - -`__.sql` - -### Versioning Format - -Bytebase supports both semantic versioning and simple timestamp-based versions. The version number is crucial for ordering migrations correctly. - -The version must begin with a number. A `v` or `V` prefix is optional. - -**Valid Version Examples:** - -- `202501150900_add_user_table.sql` -- `v1.2.3_description.sql` -- `V2_add_users_table.sql` -- `1.0_initial_schema.sql` - -### Change Type Suffixes - -You can add a suffix to the filename to specify the change type. **If the suffix is omitted, the file will be treated as the default `DDL` type.** The suffix is added to the end of the filename, just before the `.sql` extension. - -- **DDL (Default)** - - - Used for standard schema changes (Data Definition Language). This is the default type used when no suffix is present. - - **Example**: `v1.0_create_table.sql` - -- **DML** - - - Used for data manipulations (Data Manipulation Language). - - Add the `_dml` suffix. - - **Example**: `v1.0_insert_data_dml.sql` - -- **Ghost** - - Used for schema changes performed using the `gh-ost` tool. - - Add the `_ghost` suffix. - - **Example**: `v1.0_alter_table_ghost.sql` \ No newline at end of file diff --git a/mintlify/gitops/migration-based-workflow/develop.mdx b/mintlify/gitops/migration-based-workflow/develop.mdx new file mode 100644 index 000000000..166c9dc90 --- /dev/null +++ b/mintlify/gitops/migration-based-workflow/develop.mdx @@ -0,0 +1,189 @@ +--- +title: Develop +--- + +Create SQL migration files following naming conventions that enable proper version tracking and execution order. + +## File Naming Convention + +Migration filenames must follow this structure: + +``` +___.sql +``` + +**Components:** + +1. **Version** (required) - Must begin with a number, optional `v` or `V` prefix +2. **Double underscore** (`__`) separator +3. **Description** - Human-readable description using underscores or hyphens +4. **Suffix** (optional) - Migration type indicator +5. **`.sql`** file extension + +## Version Formats + +Choose a versioning strategy that fits your team: + + + + **Timestamp-Based** - Recommended for teams with parallel development + + ``` + 20250120143000__add_user_email.sql + 20250121091500__create_orders_table.sql + ``` + + Format: `YYYYMMDDHHmmss` + + ✅ No merge conflicts + ✅ Chronological ordering + ✅ Supports distributed teams + + ⚠️ Less human-readable + + + + **Semantic Versioning** - Meaningful version numbers + + ``` + v1.0.0__initial_release.sql + v1.1.0__add_user_profiles.sql + v1.1.1__fix_profile_constraint.sql + v2.0.0__redesign_authentication.sql + ``` + + ✅ Conveys change significance + ✅ Aligns with application versioning + + ⚠️ Requires coordination + ⚠️ Merge conflicts possible + + + + **Simple Sequential** - Easy to understand + + ``` + 001__initial_schema.sql + 002__add_users.sql + 003__add_products.sql + ``` + + ✅ Simple and clear + ✅ Milestones mark releases + + ⚠️ Merge conflicts in parallel work + + + +## Change Type Suffixes + +Specify the migration type using an optional suffix: + +| Suffix | Type | Description | Use Cases | +|--------|------|-------------|-----------| +| *(none)* | **DDL** | Data Definition Language | CREATE, ALTER, DROP tables, indexes, constraints | +| `_dml` | **DML** | Data Manipulation Language | INSERT, UPDATE, DELETE, data migrations | +| `_ghost` | **Ghost** | gh-ost online migration | Zero-downtime MySQL schema changes | + +**Examples:** + +``` +v1.0.0__create_schema.sql (DDL - default) +v1.1.0__seed_initial_data_dml.sql (DML - data changes) +v1.2.0__add_user_index_ghost.sql (Ghost - online schema change) +``` + + + Learn about gh-ost for zero-downtime MySQL migrations + + +## File Organization + +**Recommended structure:** + +``` +migrations/ +├── 001__initial_schema.sql +├── 002__add_users.sql +├── 003__add_products_dml.sql +└── 004__add_indexes.sql +``` + +**Or with subdirectories:** + +``` +migrations/ +├── baseline/ +│ └── 000__initial_schema.sql +├── features/ +│ ├── 001__users.sql +│ ├── 002__products.sql +│ └── 003__orders.sql +└── hotfixes/ + └── 004__fix_index.sql +``` + +## Best Practices + +**Keep migrations small and focused:** + +```sql +✅ Good - Single, clear purpose +-- 005__add_user_email.sql +ALTER TABLE users ADD COLUMN email TEXT; +``` + +```sql +❌ Bad - Too large and unfocused +-- 005__big_refactor.sql +ALTER TABLE users ADD COLUMN email TEXT; +ALTER TABLE users ADD COLUMN phone TEXT; +CREATE TABLE user_preferences (...); +-- 500 more lines +``` + +**Add comments for complex logic:** + +```sql +-- 020__migrate_legacy_permissions_dml.sql +-- Migrates old role system to new permission model +-- Old: roles.name -> New: permissions.scope + permissions.action + +UPDATE permissions +SET scope = CASE + WHEN roles.name = 'admin' THEN 'database' + WHEN roles.name = 'editor' THEN 'database' +END +FROM roles +WHERE permissions.role_id = roles.id; +``` + +**Use transactions when possible:** + +```sql +-- 015__multi_step_migration_dml.sql +BEGIN; + +UPDATE users SET status = 'migrated' WHERE status = 'legacy'; + +INSERT INTO user_audit_log (user_id, action) +SELECT id, 'status_migrated' +FROM users +WHERE status = 'migrated'; + +COMMIT; +``` + +--- + +## Next Steps + + + + Set up automated validation in your CI/CD pipeline + + + + Deploy your migrations to databases + + diff --git a/mintlify/gitops/migration-based-workflow/limitations.mdx b/mintlify/gitops/migration-based-workflow/limitations.mdx new file mode 100644 index 000000000..7581e96b6 --- /dev/null +++ b/mintlify/gitops/migration-based-workflow/limitations.mdx @@ -0,0 +1,82 @@ +--- +title: Limitations +--- + +While migration-based workflow is powerful and flexible, be aware of these considerations: + +## Version Conflicts + +**Issue:** Multiple developers creating migrations with the same version number. + +**Solution:** Use timestamp-based versioning (`YYYYMMDDHHmmss`) to avoid conflicts. + +## Migration Order Dependencies + +**Issue:** Complex dependencies between migrations can be hard to manage. + +**Solution:** +- Keep migrations small and focused +- Use descriptive names +- Document dependencies in comments +- Consider state-based workflow for pure schema changes + +## Rollback Complexity + +**Issue:** Reverting migrations requires writing reverse scripts. + +**Solution:** +- Plan rollback strategy before deploying +- Test rollback scripts in staging +- Consider using Bytebase's data rollback feature for DML changes + + + Automatic rollback for INSERT/UPDATE/DELETE operations + + +## Schema Drift Detection + +**Issue:** Manual changes bypass GitOps and cause drift. + +**Solution:** +- Restrict direct database access +- Enable drift detection +- Enforce all changes through GitOps + + + Detect and manage schema drift + + +## Large-Scale Deployments + +**Issue:** Deploying to hundreds or thousands of databases takes time. + +**Solution:** +- Use database groups for fleet management +- Configure parallel execution limits +- Enable sampling for validation (check N random databases) + + + Production-ready workflow patterns and optimizations + + +--- + +## Next Steps + + + + Learn about the declarative alternative + + + + Production-ready workflow patterns + + + + Solutions for common issues + + + + Return to GitOps overview + + diff --git a/mintlify/gitops/migration-based-workflow/overview.mdx b/mintlify/gitops/migration-based-workflow/overview.mdx new file mode 100644 index 000000000..4c30ab331 --- /dev/null +++ b/mintlify/gitops/migration-based-workflow/overview.mdx @@ -0,0 +1,78 @@ +--- +title: Overview +--- + +The migration-based workflow is the **imperative** approach to database schema management, where you write incremental SQL files that explicitly describe each change to apply. This is the traditional approach familiar to database administrators and developers using tools like Flyway, Liquibase, or Rails migrations. + +## How It Works + +Each migration file contains explicit DDL or DML statements: + +```sql +-- migrations/001__create_users.sql +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + username TEXT NOT NULL UNIQUE +); + +-- migrations/002__add_email.sql +ALTER TABLE users ADD COLUMN email TEXT; + +-- migrations/003__seed_admin_dml.sql +INSERT INTO users (username, email) VALUES ('admin', 'admin@example.com'); +``` + +Bytebase executes these files sequentially based on version numbers and tracks which versions have been applied to each database through the revision system, ensuring migrations are never re-executed. + +## When to Use Migration-Based Workflow + +✅ **Use migration-based workflow when:** + +- You need data migrations (INSERT, UPDATE, DELETE operations) +- You require precise control over the execution order +- Your team is familiar with traditional migration tools +- You're working with any database type (MySQL, PostgreSQL, SQL Server, MongoDB, etc.) +- You need to perform complex multi-step transformations +- You want explicit versioning of each database change + +## Complete Workflow + +The migration-based workflow consists of three stages: + +``` +1. Develop → 2. SQL Review (PR/MR) → 3. Release (Bytebase) +``` + +### Stage 1: Develop + +Write migration files following the naming convention and commit to your repository. + +### Stage 2: SQL Review (PR/MR) + +Open a pull/merge request. Automated SQL review runs in CI/CD. + +### Stage 3: Release (Bytebase) + +After merge, CI/CD triggers Bytebase to create a release and deploy to target databases. + +--- + +## Next Steps + + + + Learn about file naming conventions and versioning strategies + + + + Set up automated SQL validation in your CI/CD pipeline + + + + Deploy migrations to your databases + + + + Understand constraints and considerations + + diff --git a/mintlify/gitops/migration-based-workflow/release.mdx b/mintlify/gitops/migration-based-workflow/release.mdx new file mode 100644 index 000000000..e92c1f34a --- /dev/null +++ b/mintlify/gitops/migration-based-workflow/release.mdx @@ -0,0 +1,193 @@ +--- +title: Release +--- + +After PR/MR approval and merge, your CI/CD pipeline triggers Bytebase to create a release and deploy the migrations. + +## How Releases Work + +A **Release** is an immutable package containing all your SQL migration files: + +- Linked to VCS commit for full traceability +- Files validated and stored with SHA256 checksums +- Can be deployed to multiple environments +- Supports progressive rollout strategies + +**Inside Bytebase, the release triggers:** + +1. **Plan Generation** - Defines target databases and rollout strategy +2. **Rollout Execution** - Creates tasks for each database +3. **Revision Tracking** - Records applied migrations to prevent duplicates + +## CI/CD Integration Examples + + + + ```yaml + # .github/workflows/release.yml + name: Database Release + on: + push: + branches: [main] + paths: + - 'migrations/**' + + jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create Bytebase Release + uses: bytebase/sql-review-action@v1 + with: + command: rollout + bytebase-url: ${{ secrets.BYTEBASE_URL }} + bytebase-token: ${{ secrets.BYTEBASE_SERVICE_ACCOUNT_TOKEN }} + file-pattern: 'migrations/**/*.sql' + project: projects/my-project + targets: instances/prod/databases/mydb + ``` + + + + + + + + ```yaml + # .gitlab-ci.yml + rollout: + image: bytebase/bytebase-action:latest + stage: deploy + only: + - main + script: + - | + bytebase-action rollout \ + --url $BYTEBASE_URL \ + --service-account $BYTEBASE_SERVICE_ACCOUNT \ + --service-account-secret $BYTEBASE_SERVICE_ACCOUNT_SECRET \ + --file-pattern "migrations/**/*.sql" \ + --project projects/my-project \ + --targets instances/prod/databases/mydb + ``` + + + + + + + + ```yaml + # azure-pipelines.yml + trigger: + branches: + include: + - main + paths: + include: + - migrations/** + + jobs: + - job: Release + pool: + vmImage: 'ubuntu-latest' + steps: + - task: Docker@2 + inputs: + command: run + arguments: > + -e BYTEBASE_URL=$(BYTEBASE_URL) + -e BYTEBASE_SERVICE_ACCOUNT=$(BYTEBASE_SERVICE_ACCOUNT) + -e BYTEBASE_SERVICE_ACCOUNT_SECRET=$(BYTEBASE_SERVICE_ACCOUNT_SECRET) + -v $(System.DefaultWorkingDirectory):/workspace + bytebase/bytebase-action + rollout + --file-pattern "/workspace/migrations/**/*.sql" + --project projects/my-project + --targets instances/prod/databases/mydb + ``` + + + + + + + + ```yaml + # bitbucket-pipelines.yml + pipelines: + branches: + main: + - step: + name: Database Release + image: bytebase/bytebase-action:latest + deployment: production + script: + - | + bytebase-action rollout \ + --url $BYTEBASE_URL \ + --service-account $BYTEBASE_SERVICE_ACCOUNT \ + --service-account-secret $BYTEBASE_SERVICE_ACCOUNT_SECRET \ + --file-pattern "migrations/**/*.sql" \ + --project projects/my-project \ + --targets instances/prod/databases/mydb + ``` + + + + + + + +## Deployment Strategies + +**Progressive rollout across environments:** + +```yaml +# Deploy to dev, then staging, then production +targets: > + instances/dev/databases/mydb, + instances/staging/databases/mydb, + instances/prod/databases/mydb +``` + +**Multi-tenant deployment using database groups:** + +```yaml +# Deploy to entire production fleet +targets: projects/my-project/databaseGroups/production-fleet +``` + + + Manage database fleets with groups + + +## Idempotency Guarantee + +Bytebase tracks which migration versions have been applied to each database via the **revision system**. When creating a release: + +1. **Check revisions** - Query which versions already exist +2. **Skip applied** - Migrations with matching versions are skipped +3. **Execute new** - Only unapplied versions are executed +4. **Record success** - Create revision only on successful completion + +This ensures: +- ✅ Safe to run the same release multiple times +- ✅ Migrations never double-execute +- ✅ Environment parity (dev migrations auto-skip in prod) + +--- + +## Next Steps + + + + Understand constraints and considerations + + + + Production-ready workflow patterns + + diff --git a/mintlify/gitops/migration-based-workflow/sql-review-ci.mdx b/mintlify/gitops/migration-based-workflow/sql-review-ci.mdx new file mode 100644 index 000000000..fafda859c --- /dev/null +++ b/mintlify/gitops/migration-based-workflow/sql-review-ci.mdx @@ -0,0 +1,165 @@ +--- +title: SQL Review CI +--- + +Automated SQL review validates your migration files during pull/merge requests, catching issues before they reach production. + +## What Gets Validated + +**Automated checks include:** +- SQL syntax validation +- Policy rule enforcement (200+ configurable rules) +- Naming convention compliance +- Risk assessment for dangerous operations +- Schema compatibility checks + + + Configure 200+ linting rules for automated validation + + +## Setup + +**Prerequisites:** +1. [Set up SQL Review Policy](/sql-review/review-policy) - Define your SQL standards +2. [Install GitOps Integration](/gitops/installation) - Connect Bytebase to your VCS + +## CI/CD Integration Examples + + + + ```yaml + # .github/workflows/sql-review.yml + name: SQL Review + on: + pull_request: + paths: + - 'migrations/**' + + jobs: + sql-review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: SQL Review + uses: bytebase/sql-review-action@v1 + with: + bytebase-url: ${{ secrets.BYTEBASE_URL }} + bytebase-token: ${{ secrets.BYTEBASE_SERVICE_ACCOUNT_TOKEN }} + file-pattern: 'migrations/**/*.sql' + ``` + + + + + + ![GitHub PR SQL Review](/content/docs/gitops/sql-review-ci/github-pr-sql-review.webp) + + + + ```yaml + # .gitlab-ci.yml + sql-review: + image: bytebase/bytebase-action:latest + stage: test + only: + - merge_requests + script: + - | + bytebase-action check \ + --url $BYTEBASE_URL \ + --service-account $BYTEBASE_SERVICE_ACCOUNT \ + --service-account-secret $BYTEBASE_SERVICE_ACCOUNT_SECRET \ + --file-pattern "migrations/**/*.sql" + ``` + + + + ![GitLab PR SQL Review](/content/docs/gitops/sql-review-ci/gitlab-pr-sql-review.webp) + + + + ```yaml + # azure-pipelines.yml + trigger: none + pr: + branches: + include: + - main + paths: + include: + - migrations/** + + jobs: + - job: SQLReview + pool: + vmImage: 'ubuntu-latest' + steps: + - task: Docker@2 + inputs: + command: run + arguments: > + -e BYTEBASE_URL=$(BYTEBASE_URL) + -e BYTEBASE_SERVICE_ACCOUNT=$(BYTEBASE_SERVICE_ACCOUNT) + -e BYTEBASE_SERVICE_ACCOUNT_SECRET=$(BYTEBASE_SERVICE_ACCOUNT_SECRET) + -v $(System.DefaultWorkingDirectory):/workspace + bytebase/bytebase-action + check --file-pattern "/workspace/migrations/**/*.sql" + ``` + + + + + + ![Azure DevOps PR SQL Review](/content/docs/gitops/sql-review-ci/azure-devops-pr-sql-review.webp) + + + + ```yaml + # bitbucket-pipelines.yml + pipelines: + pull-requests: + '**': + - step: + name: SQL Review + image: bytebase/bytebase-action:latest + script: + - | + bytebase-action check \ + --url $BYTEBASE_URL \ + --service-account $BYTEBASE_SERVICE_ACCOUNT \ + --service-account-secret $BYTEBASE_SERVICE_ACCOUNT_SECRET \ + --file-pattern "migrations/**/*.sql" + ``` + + + + + + ![Bitbucket PR SQL Review](/content/docs/gitops/sql-review-ci/bb-pr-sql-review.webp) + + + +## Review Bot Feedback + +The SQL review bot posts detailed feedback as PR/MR comments: + +- ✅ **Passed checks** - Migration meets all policy requirements +- ⚠️ **Warnings** - Best practice violations (non-blocking by default) +- ❌ **Errors** - Policy violations (blocking by default) +- 📊 **Risk assessment** - Evaluation of potential impact +- 📝 **Detailed explanations** - Why rules failed with fix suggestions + +--- + +## Next Steps + + + + Deploy your migrations after review approval + + + + Explore all available SQL review rules + + diff --git a/mintlify/gitops/overview.mdx b/mintlify/gitops/overview.mdx index e46c87df8..c92c5ce66 100644 --- a/mintlify/gitops/overview.mdx +++ b/mintlify/gitops/overview.mdx @@ -36,26 +36,147 @@ Bytebase enables database-as-code workflows, allowing you to manage database cha ## GitOps Workflow -### 1. Development Phase +Bytebase GitOps follows a streamlined 3-stage pipeline: -Developers create SQL migration files in feature branches, following the same branch management strategy as application code (e.g., [GitHub Flow](https://docs.github.com/en/get-started/quickstart/github-flow), [GitLab Flow](https://about.gitlab.com/blog/gitlab-flow-duo/)). +``` +Develop → Review (PR/MR) → Release (Bytebase) +``` -### 2. Review Phase (Pull Request) +### 1. Develop -- Automated SQL review validates changes against configured policies -- Schema changes are reviewed alongside application code changes -- Compatibility checks ensure schema and code alignment +Developers create SQL files in feature branches following your team's Git workflow (GitHub Flow, GitLab Flow, trunk-based, etc.). -### 3. Release Creation and Deployment +**What happens:** +- Write SQL migration or schema files +- Follow naming conventions for proper versioning +- Test changes locally when possible +- Commit to version control -After PR merge, releases can be created and deployed according to your workflow, consistent with application code deployment: +### 2. Review (Pull Request / Merge Request) -- GitOps integration can create versioned releases containing SQL migrations -- Releases are linked to commits for full traceability -- Deployment timing is configurable based on your pipeline strategy: - - **Automatic**: Deploy immediately after merge (common for development/test environments) - - **Time-based**: Schedule deployments for specific windows - - **Manual approval**: Require explicit approval before deployment (common for production) -- Feature flags can decouple schema deployment from applications using new schema version +When developers open a pull/merge request, automated validation ensures changes meet your organization's standards. -**Key Principle**: Bytebase automatically detects previously applied migrations and skips them, ensuring safe re-deployment and idempotent operations across all environments. +**What happens:** +- **Automated SQL Review** - Validates syntax and enforces policies +- **Risk Assessment** - Identifies high-risk operations +- **Schema Compatibility** - Checks for breaking changes +- **Team Review** - Human review of schema changes alongside code + +### 3. Release (Bytebase) + +After PR/MR approval and merge, Bytebase orchestrates the deployment. This single stage encompasses **Plan → Rollout → Revision** tracking, all managed within Bytebase. + +**What happens in Bytebase:** + +1. **Release Creation** - Immutable package of SQL files linked to VCS commit +2. **Plan Generation** - Defines deployment strategy and target databases +3. **Rollout Execution** - Creates and executes tasks across environments +4. **Revision Tracking** - Records applied migrations to prevent duplicates + + +The Release stage in Bytebase involves several coordinated steps: + +- **Plan**: Defines which databases to target and rollout strategy +- **Rollout**: Executes the plan, creating stages for progressive deployment +- **Revision**: Tracks which migrations have been applied to each database + +For detailed information about these components, see: +- Plan creation and targeting strategies +- Rollout execution and approval workflows +- Revision-based idempotency + +These topics are covered in detail within each workflow documentation. + + +**Key Principle**: Bytebase automatically detects previously applied changes by checking revision history and skips them, ensuring safe re-deployment and idempotent operations across all environments. + +## Two Workflow Approaches + +Bytebase supports two complementary approaches for managing database schema changes. Choose the one that fits your team's needs, or use both for different scenarios. + +### Migration-Based Workflow + +The **imperative** approach where you write incremental change scripts that describe how to modify the schema. + +```sql +-- migrations/001__create_users.sql +CREATE TABLE users (id INT PRIMARY KEY, name TEXT); + +-- migrations/002__add_email.sql +ALTER TABLE users ADD COLUMN email TEXT; +``` + +**Best for:** +- Teams familiar with traditional database migration tools +- Complex changes requiring specific execution order +- Data migrations (INSERT, UPDATE, DELETE operations) +- All supported databases (MySQL, PostgreSQL, SQL Server, etc.) +- Situations requiring fine-grained control over each change + +**How it works:** Each migration file contains explicit DDL/DML statements. Files execute sequentially based on version numbers. Bytebase tracks which versions have been applied to prevent re-execution. + + + Complete guide to migration-based approach with examples and best practices + + +### State-Based Workflow (SDL) + +The **declarative** approach where you define the desired end-state of your database schema, and Bytebase automatically generates the migration DDL. + +```sql +-- schema/public.sql (complete desired state) +CREATE TABLE public.users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + email TEXT +); +``` + +**Best for:** +- Teams adopting infrastructure-as-code principles +- Pure schema changes (DDL only, no data operations) +- Simplified Git diffs showing schema evolution +- PostgreSQL databases (currently) +- Reducing complexity around migration ordering + +**How it works:** You maintain the complete schema definition. Bytebase compares your desired state with the current database state and automatically generates the necessary ALTER/DROP statements to reach the desired state. + + + Complete guide to state-based (SDL) approach with validation rules and examples + + +## Choosing Between Workflows + +| Aspect | Migration-Based | State-Based (SDL) | +|--------|-----------------|-------------------| +| **Approach** | Imperative (how to change) | Declarative (desired end state) | +| **Files** | Multiple migration scripts | Single schema definition per module | +| **Git History** | Shows individual changes | Shows current state + diffs | +| **Operations** | DDL + DML | DDL only | +| **Data Changes** | ✅ Supported | ❌ Not supported | +| **Database Support** | All databases | PostgreSQL only (currently) | +| **Learning Curve** | Familiar to DBAs | Familiar to DevOps/IaC users | +| **Control** | Explicit control over each step | Automatic diff generation | +| **Rollback** | Write reverse migration | Define previous state | + +**Can I use both?** Yes! You can use state-based workflow for schema structure and migration-based workflow for data operations in the same project. + +## Next Steps + + + + Traditional approach with incremental changes + + + + Modern declarative schema management + + + + Set up VCS integration and runners + + + + Production-ready workflow patterns + + diff --git a/mintlify/gitops/release.mdx b/mintlify/gitops/release.mdx deleted file mode 100644 index 6ed776ecf..000000000 --- a/mintlify/gitops/release.mdx +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Release ---- - -Bytebase integrates with popular CI/CD platforms to automate database releases through GitOps workflows. When triggered by your CI/CD pipeline, Bytebase creates a **release** - an immutable package containing all your SQL migration files that can be progressively deployed across environments. - -## Setup - -Before creating releases, you'll need to [install the GitOps integration](/gitops/installation). - - -## Examples - -We provide examples for GitHub, GitLab, Bitbucket, and Azure DevOps below. Adjust the parameters to match your repository structure and rollout workflow. - -## GitHub - - - - - -## GitLab - - - - - -## Bitbucket - - - - - -## Azure DevOps - - - - \ No newline at end of file diff --git a/mintlify/gitops/sql-review-ci.mdx b/mintlify/gitops/sql-review-ci.mdx deleted file mode 100644 index 9e6b11f82..000000000 --- a/mintlify/gitops/sql-review-ci.mdx +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: SQL Review CI ---- - -SQL Review CI validates your SQL changes against configured policies during pull/merge requests. The review bot posts feedback directly as PR comments. - -## Setup - -Before enabling SQL Review CI: - -1. [Set up SQL Review Policy](/sql-review/review-policy) - Define your SQL standards and rules -2. [Install GitOps Integration](/gitops/installation) - Connect Bytebase to your VCS - -## Examples - -We provide examples for GitHub, GitLab, Bitbucket, and Azure DevOps below. Adjust the parameters to match your repository structure and workflow. - -## GitHub - - - - - -![github](/content/docs/gitops/sql-review-ci/github-pr-sql-review.webp) - -## GitLab - - - -![gitlab](/content/docs/gitops/sql-review-ci/gitlab-pr-sql-review.webp) - -## Bitbucket - - - -![bitbucket](/content/docs/gitops/sql-review-ci/bb-pr-sql-review.webp) - -## Azure DevOps - - - -![azure devops](/content/docs/gitops/sql-review-ci/azure-devops-pr-sql-review.webp) \ No newline at end of file diff --git a/mintlify/gitops/state-based-workflow/develop.mdx b/mintlify/gitops/state-based-workflow/develop.mdx new file mode 100644 index 000000000..d189be591 --- /dev/null +++ b/mintlify/gitops/state-based-workflow/develop.mdx @@ -0,0 +1,230 @@ +--- +title: Develop +--- + +Create SDL files that define your complete database schema in a declarative format. + +## Getting Started + +When starting with state-based workflow: + +1. **Export Current Schema** - In Bytebase database detail page, click `Export Schema` to download your current database schema +2. **Organize Schema Files** - Edit and organize the exported schema into manageable files +3. **Commit to Repository** - Add SDL files to version control +4. **Make Changes** - Update the desired state and commit + +## File Organization + +Organize SDL files by schema or module: + +``` +schema/ +├── public.sql # Public schema objects +├── analytics.sql # Analytics schema +└── internal.sql # Internal schema +``` + +Or split by object type: + +``` +schema/ +├── 01_tables.sql +├── 02_indexes.sql +├── 03_views.sql +├── 04_functions.sql +└── 05_sequences.sql +``` + +## SDL Syntax Requirements + +SDL enforces strict conventions to ensure maintainability: + +### 1. Schema Qualification Required + +All objects must include schema prefix: + +```sql +✅ Correct - Fully qualified names +CREATE TABLE public.users (...); +CREATE INDEX idx_name ON public.users(name); +CREATE FUNCTION public.get_user(...) RETURNS ...; +``` + +```sql +❌ Incorrect - Missing schema +CREATE TABLE users (...); +CREATE INDEX idx_name ON users(name); +``` + +### 2. Table-Level Constraints + +PRIMARY KEY, UNIQUE, FOREIGN KEY, and CHECK constraints must be defined at table level with explicit names: + +```sql +✅ Correct - Table-level with names +CREATE TABLE public.users ( + id INTEGER NOT NULL, -- NOT NULL allowed at column level + email TEXT NOT NULL, -- NOT NULL allowed at column level + created_at TIMESTAMP DEFAULT NOW(), -- DEFAULT allowed at column level + CONSTRAINT users_pkey PRIMARY KEY (id), + CONSTRAINT users_email_key UNIQUE (email), + CONSTRAINT users_email_check CHECK (email LIKE '%@%') +); +``` + +```sql +❌ Incorrect - Column-level constraints +CREATE TABLE public.users ( + id INTEGER PRIMARY KEY, -- Must be at table level + email TEXT UNIQUE, -- Must be at table level + age INTEGER CHECK (age >= 0) -- Must be at table level +); +``` + +**Allowed at column level:** +- `NOT NULL` +- `DEFAULT` +- `GENERATED ALWAYS AS` +- `SERIAL` / `BIGSERIAL` + +### 3. Named Constraints + +All table constraints require explicit names: + +```sql +✅ Correct - Named constraints +CONSTRAINT users_pkey PRIMARY KEY (id) +CONSTRAINT users_email_key UNIQUE (email) +CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES public.users(id) +CONSTRAINT check_positive_age CHECK (age > 0) +``` + +```sql +❌ Incorrect - Unnamed constraints +PRIMARY KEY (id) +UNIQUE (email) +FOREIGN KEY (user_id) REFERENCES public.users(id) +CHECK (age > 0) +``` + +**Naming conventions:** +- Primary keys: `{table}_pkey` +- Unique constraints: `{table}_{column}_key` +- Foreign keys: `fk_{table}_{referenced_table}` +- Check constraints: `check_{description}` or `{table}_{column}_check` + +### 4. Foreign Key References + +Foreign keys must use fully qualified table names: + +```sql +✅ Correct - Fully qualified reference +CREATE TABLE public.orders ( + id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + CONSTRAINT orders_pkey PRIMARY KEY (id), + CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES public.users(id) +); +``` + +```sql +❌ Incorrect - Unqualified reference +CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES users(id) +``` + +### 5. Named Indexes + +All indexes must have explicit names: + +```sql +✅ Correct - Named indexes +CREATE INDEX idx_users_email ON public.users(email); +CREATE UNIQUE INDEX idx_users_username ON public.users(username); +``` + +```sql +❌ Incorrect - Unnamed index +CREATE INDEX ON public.users(email); +``` + +## Supported Statements + +SDL files support these PostgreSQL statements: + +- `CREATE TABLE` +- `CREATE INDEX` / `CREATE UNIQUE INDEX` +- `CREATE VIEW` +- `CREATE SEQUENCE` +- `CREATE FUNCTION` +- `ALTER SEQUENCE ... OWNED BY` (for serial columns) + +**Not Allowed:** +- `ALTER TABLE`, `DROP` (Bytebase generates these) +- `INSERT`, `UPDATE`, `DELETE` (use migration-based workflow) +- Transaction control (`BEGIN`, `COMMIT`) + +## Complete SDL Example + +```sql +-- schema/public.sql + +-- Sequences +CREATE SEQUENCE public.users_id_seq; + +-- Tables +CREATE TABLE public.users ( + id INTEGER NOT NULL DEFAULT nextval('public.users_id_seq'::regclass), + username TEXT NOT NULL, + email TEXT NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + status TEXT NOT NULL DEFAULT 'active', + CONSTRAINT users_pkey PRIMARY KEY (id), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_email_key UNIQUE (email), + CONSTRAINT check_email_format CHECK (email ~* '^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}$'), + CONSTRAINT check_status_values CHECK (status IN ('active', 'inactive', 'suspended')) +); + +ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; + +CREATE TABLE public.user_profiles ( + user_id INTEGER NOT NULL, + bio TEXT, + avatar_url TEXT, + CONSTRAINT user_profiles_pkey PRIMARY KEY (user_id), + CONSTRAINT fk_user_profiles_users FOREIGN KEY (user_id) REFERENCES public.users(id) ON DELETE CASCADE +); + +-- Indexes +CREATE INDEX idx_users_email ON public.users(email); +CREATE INDEX idx_users_created_at ON public.users(created_at); +CREATE INDEX idx_users_status ON public.users(status) WHERE status != 'inactive'; + +-- Views +CREATE VIEW public.active_users AS +SELECT id, username, email, created_at +FROM public.users +WHERE status = 'active'; + +-- Functions +CREATE FUNCTION public.get_user_count() +RETURNS INTEGER +LANGUAGE sql +AS $$ + SELECT COUNT(*) FROM public.users; +$$; +``` + +--- + +## Next Steps + + + + Set up SDL validation in your CI/CD pipeline + + + + Deploy your SDL changes to databases + + diff --git a/mintlify/gitops/state-based-workflow/limitations.mdx b/mintlify/gitops/state-based-workflow/limitations.mdx new file mode 100644 index 000000000..b06b43bd1 --- /dev/null +++ b/mintlify/gitops/state-based-workflow/limitations.mdx @@ -0,0 +1,166 @@ +--- +title: Limitations +--- + +State-based workflow (SDL) currently has these limitations: + +## Database Support + +**Supported:** +- PostgreSQL +- CockroachDB +- Redshift (PostgreSQL-compatible) + +**Not yet supported:** +- MySQL +- SQL Server +- Oracle +- MongoDB +- Other databases + +## Supported SQL Statements + +Only these PostgreSQL statements are supported: + +- `CREATE TABLE` +- `CREATE INDEX` / `CREATE UNIQUE INDEX` +- `CREATE VIEW` +- `CREATE SEQUENCE` +- `CREATE FUNCTION` +- `ALTER SEQUENCE` (for OWNED BY) + +**Not supported:** +- Complex stored procedures +- Triggers +- Row-level security policies +- DML operations (INSERT, UPDATE, DELETE) +- Transaction control +- Database-level settings + +## Strict Syntax Requirements + +SDL requires strict adherence to conventions: + +1. **All objects must use fully qualified names** (with schema prefix) + ```sql + -- Required: public.users + -- Not allowed: users + ``` + +2. **PRIMARY KEY, UNIQUE, FOREIGN KEY, CHECK must be table-level with explicit names** + ```sql + -- Correct: + CONSTRAINT users_pkey PRIMARY KEY (id) + + -- Not allowed: + id INTEGER PRIMARY KEY + ``` + +3. **Only NOT NULL, DEFAULT, GENERATED allowed at column level** + ```sql + -- Allowed: + id SERIAL, + name TEXT NOT NULL, + created_at TIMESTAMP DEFAULT NOW() + + -- Not allowed: + id INTEGER PRIMARY KEY + ``` + +4. **Foreign key references must be fully qualified** + ```sql + -- Required: + REFERENCES public.users(id) + + -- Not allowed: + REFERENCES users(id) + ``` + +5. **All indexes must have explicit names** + ```sql + -- Required: + CREATE INDEX idx_users_email ON public.users(email) + + -- Not allowed: + CREATE INDEX ON public.users(email) + ``` + +## No Direct Data Operations + +SDL only manages schema structure. For data operations, use migration-based workflow. + +**Not supported in SDL:** +- INSERT statements +- UPDATE statements +- DELETE statements +- Data transformation logic + +**Solution:** Combine both workflows: +``` +schema/ # SDL for schema structure + ├── tables.sql + └── indexes.sql +migrations/ # Migration-based for data + ├── 001__seed_roles_dml.sql + └── 002__migrate_users_dml.sql +``` + +## Destructive Operations + +SDL-generated DROP statements execute automatically when objects are removed from files: + +```sql +-- Remove table from SDL file +-- → Bytebase generates: DROP TABLE old_table; +``` + + + Always backup data before deploying SDL changes that remove objects from schema files. Bytebase will automatically drop those objects. + + +## Limited Rollback + +SDL only moves forward to new desired states: + +- No automatic rollback generation +- To rollback: revert SDL files to previous state and redeploy +- Data in dropped objects is lost (backup required) + + + Approaches for reversing schema changes + + +## Performance Considerations + +For large schemas: + +- Initial SDL adoption requires exporting complete schema +- State comparison time increases with schema complexity +- DDL generation uses topological sort (handles dependencies) + +**Mitigation:** +- Organize schema into multiple files +- Use database groups for fleet management +- Test SDL workflow on staging first + +--- + +## Next Steps + + + + Learn about the imperative alternative + + + + Production-ready workflow patterns + + + + Solutions for common issues + + + + Return to GitOps overview + + diff --git a/mintlify/gitops/state-based-workflow/overview.mdx b/mintlify/gitops/state-based-workflow/overview.mdx new file mode 100644 index 000000000..fb29b5419 --- /dev/null +++ b/mintlify/gitops/state-based-workflow/overview.mdx @@ -0,0 +1,90 @@ +--- +title: Overview +--- + +The state-based workflow uses **SDL (Schema Definition Language)** - a declarative approach where you define the desired end-state of your database schema, and Bytebase automatically generates the migration DDL. This approach mirrors infrastructure-as-code principles used by Kubernetes, Terraform, and other modern DevOps tools. + +## How It Works + +Instead of writing incremental changes, you maintain the complete schema definition: + +```sql +-- schema/public.sql (complete desired state) +CREATE TABLE public.users ( + id INTEGER, + name TEXT NOT NULL, + email TEXT, + CONSTRAINT users_pkey PRIMARY KEY (id), + CONSTRAINT users_email_key UNIQUE (email) +); + +CREATE INDEX idx_users_email ON public.users(email); +``` + +When you update this file and deploy, Bytebase: +1. Compares your desired state with the current database +2. Automatically generates the necessary ALTER/DROP statements +3. Executes the generated migration to reach the desired state + +## When to Use State-Based Workflow + +✅ **Use state-based workflow when:** + +- You're managing pure schema changes (DDL only) +- Your team embraces infrastructure-as-code principles +- You want Git-friendly diffs showing schema evolution +- You need simplified management without tracking migration order +- You're working with PostgreSQL databases (currently supported) +- You prefer automatic dependency resolution over manual ordering + +❌ **Don't use state-based workflow when:** + +- You need data migrations (INSERT, UPDATE, DELETE) +- You require complex multi-step logic +- You're working with MySQL, SQL Server, or other databases (not yet supported) + + + **Hybrid Approach:** Use state-based workflow for schema structure and migration-based workflow for data operations in the same project. + + +## Complete Workflow + +The state-based workflow follows the same three stages: + +``` +1. Develop → 2. SQL Review (PR/MR) → 3. Release (Bytebase) +``` + +### Stage 1: Develop + +Maintain complete schema definition files representing desired state. + +### Stage 2: SQL Review (PR/MR) + +Open a pull/merge request. SDL validation runs in CI/CD. + +### Stage 3: Release (Bytebase) + +After merge, Bytebase compares states, generates DDL, and deploys. + +--- + +## Next Steps + + + + Learn about SDL syntax and requirements + + + + Set up SDL validation in your CI/CD pipeline + + + + Deploy SDL changes to your databases + + + + Understand constraints and considerations + + diff --git a/mintlify/gitops/state-based-workflow/release.mdx b/mintlify/gitops/state-based-workflow/release.mdx new file mode 100644 index 000000000..440cc32fe --- /dev/null +++ b/mintlify/gitops/state-based-workflow/release.mdx @@ -0,0 +1,145 @@ +--- +title: Release +--- + +After PR/MR merge, Bytebase creates a release, compares the desired state with current database state, generates migration DDL, and deploys. + +## How SDL Release Works + +1. **State Comparison** - Bytebase compares your SDL files with current database schema +2. **DDL Generation** - Automatically generates ALTER/DROP statements using topological sort +3. **Migration Execution** - Applies generated DDL to reach desired state +4. **Revision Tracking** - Records new version for future comparisons + +## CI/CD Integration + +Add `--declarative` flag to enable SDL mode: + + + + ```yaml + # .github/workflows/release.yml + name: SDL Release + on: + push: + branches: [main] + paths: + - 'schema/**' + + jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Deploy SDL + uses: bytebase/sql-review-action@v1 + with: + command: rollout + bytebase-url: ${{ secrets.BYTEBASE_URL }} + bytebase-token: ${{ secrets.BYTEBASE_SERVICE_ACCOUNT_TOKEN }} + file-pattern: 'schema/**/*.sql' + project: projects/my-project + targets: instances/prod/databases/mydb + declarative: true + ``` + + + + ```yaml + # .gitlab-ci.yml + sdl-rollout: + image: bytebase/bytebase-action:latest + stage: deploy + only: + - main + script: + - | + bytebase-action rollout \ + --url $BYTEBASE_URL \ + --service-account $BYTEBASE_SERVICE_ACCOUNT \ + --service-account-secret $BYTEBASE_SERVICE_ACCOUNT_SECRET \ + --file-pattern "schema/**/*.sql" \ + --project projects/my-project \ + --targets instances/prod/databases/mydb \ + --declarative + ``` + + + + ```yaml + # azure-pipelines.yml + jobs: + - job: SDLRelease + pool: + vmImage: 'ubuntu-latest' + steps: + - task: Docker@2 + inputs: + command: run + arguments: > + bytebase/bytebase-action + rollout + --file-pattern "/workspace/schema/**/*.sql" + --project projects/my-project + --targets instances/prod/databases/mydb + --declarative + ``` + + + +## Version Management + +SDL automatically generates versions using timestamp format `YYYYMMDD.HHMMSS`: + +``` +Release 2025-01-15 10:30:00 → Version: 20250115.103000 +Release 2025-01-20 14:15:00 → Version: 20250120.141500 +``` + +All files in a release share the same version. Bytebase compares this version with the latest revision to determine if deployment is needed. + +## Migration Generation Example + +**Current Database:** +```sql +CREATE TABLE public.users ( + id INTEGER PRIMARY KEY, + username TEXT NOT NULL +); +``` + +**New SDL:** +```sql +CREATE TABLE public.users ( + id INTEGER, + username TEXT NOT NULL, + email TEXT NOT NULL, + CONSTRAINT users_pkey PRIMARY KEY (id), + CONSTRAINT users_email_key UNIQUE (email) +); + +CREATE INDEX idx_users_email ON public.users(email); +``` + +**Generated Migration:** +```sql +-- Bytebase automatically generates: +ALTER TABLE public.users ADD COLUMN email TEXT NOT NULL; +ALTER TABLE public.users ADD CONSTRAINT users_email_key UNIQUE (email); +CREATE INDEX idx_users_email ON public.users(email); +``` + +--- + +## Next Steps + + + + Understand SDL constraints and considerations + + + + Production-ready workflow patterns + + diff --git a/mintlify/gitops/state-based-workflow/sql-review-ci.mdx b/mintlify/gitops/state-based-workflow/sql-review-ci.mdx new file mode 100644 index 000000000..728e6cd15 --- /dev/null +++ b/mintlify/gitops/state-based-workflow/sql-review-ci.mdx @@ -0,0 +1,165 @@ +--- +title: SQL Review CI +--- + +SDL validation runs automatically during pull/merge requests to catch syntax and convention violations before merge. + + + **Work in Progress**: SDL SQL Review currently supports basic syntax checks and pre-defined SDL validation rules. Custom SQL Review rules configured in the Bytebase SQL Review Policy feature are not yet supported for SDL workflows, but this capability is actively under development. + + +## What Gets Validated + +**SDL-specific checks (Current):** +- Schema qualification on all objects +- Table-level constraint placement +- Constraint naming requirements +- Foreign key type matching +- Cross-file integrity validation +- Unsupported statement detection +- SQL syntax validation + +**Coming Soon:** +- Custom SQL Review Policy rules from Bytebase +- Team-specific naming conventions +- Custom validation rules + + + Learn about SQL Review Policy (migration-based workflow) + + +## CI/CD Integration + +Enable declarative mode by adding the `--declarative` flag: + + + + ```yaml + # .github/workflows/sql-review.yml + name: SDL Review + on: + pull_request: + paths: + - 'schema/**' + + jobs: + sdl-review: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: SDL Validation + uses: bytebase/sql-review-action@v1 + with: + bytebase-url: ${{ secrets.BYTEBASE_URL }} + bytebase-token: ${{ secrets.BYTEBASE_SERVICE_ACCOUNT_TOKEN }} + file-pattern: 'schema/**/*.sql' + declarative: true # Enable SDL mode + ``` + + + + + + ```yaml + # .gitlab-ci.yml + sdl-review: + image: bytebase/bytebase-action:latest + stage: test + only: + - merge_requests + script: + - | + bytebase-action check \ + --url $BYTEBASE_URL \ + --service-account $BYTEBASE_SERVICE_ACCOUNT \ + --service-account-secret $BYTEBASE_SERVICE_ACCOUNT_SECRET \ + --file-pattern "schema/**/*.sql" \ + --declarative + ``` + + + + + + ```yaml + # azure-pipelines.yml + jobs: + - job: SDLReview + pool: + vmImage: 'ubuntu-latest' + steps: + - task: Docker@2 + inputs: + command: run + arguments: > + bytebase/bytebase-action + check --file-pattern "/workspace/schema/**/*.sql" + --declarative + ``` + + + + + +## Common Validation Errors + + + + **Error:** `Table 'users' must include schema name` + + **Fix:** Add schema prefix to all objects + ```sql + -- Change: CREATE TABLE users + -- To: CREATE TABLE public.users + ``` + + + + **Error:** `PRIMARY KEY must be table-level constraint` + + **Fix:** Move constraint to table level + ```sql + -- Change: + id INTEGER PRIMARY KEY + + -- To: + id INTEGER, + CONSTRAINT users_pkey PRIMARY KEY (id) + ``` + + + + **Error:** `All constraints must have explicit names` + + **Fix:** Add CONSTRAINT keyword with name + ```sql + -- Change: UNIQUE (email) + -- To: CONSTRAINT users_email_key UNIQUE (email) + ``` + + + + **Error:** `Foreign key column 'user_id' (INTEGER) references 'users.id' (BIGINT)` + + **Fix:** Align column types + ```sql + -- Ensure both are same type + user_id BIGINT -- Match users.id type + ``` + + + +--- + +## Next Steps + + + + Deploy SDL changes after validation approval + + + + Explore all available SQL review rules + + diff --git a/mintlify/gitops/troubleshooting/cicd-issues.mdx b/mintlify/gitops/troubleshooting/cicd-issues.mdx new file mode 100644 index 000000000..3131628f5 --- /dev/null +++ b/mintlify/gitops/troubleshooting/cicd-issues.mdx @@ -0,0 +1,290 @@ +--- +title: CI/CD Issues +--- + +Troubleshoot problems with CI/CD integration, authentication, and automated workflows. + +## GitHub Actions: API Call Fails + +**Error:** +``` +curl: (6) Could not resolve host: bytebase.example.com +``` + +**Causes:** +1. Incorrect Bytebase URL +2. Network restrictions +3. DNS resolution issues + +**Solutions:** + +```yaml +# .github/workflows/gitops.yml + +# 1. Verify BB_URL environment variable +- name: Check Bytebase URL + run: echo "Bytebase URL: $BB_URL" + env: + BB_URL: ${{ secrets.BYTEBASE_URL }} + +# 2. Test connectivity +- name: Test Connection + run: curl -f $BB_URL/v1/actuator/info + +# 3. Use correct authentication +- name: Create Release + run: | + curl -X POST "$BB_URL/v1/projects/my-project/releases" \ + -H "Authorization: Bearer $BB_TOKEN" \ + -H "Content-Type: application/json" \ + -d @release.json + env: + BB_URL: ${{ secrets.BYTEBASE_URL }} + BB_TOKEN: ${{ secrets.BYTEBASE_TOKEN }} +``` + +## Unauthorized Error + +**Error:** +``` +401 Unauthorized: Invalid token +``` + +**Causes:** +1. Token expired or invalid +2. Token not properly set in secrets +3. Wrong token format + +**Solutions:** + + + + Test token manually: + ```bash + curl -H "Authorization: Bearer $BB_TOKEN" \ + $BB_URL/v1/actuator/info + ``` + + Should return server info, not 401. + + + + 1. Login to Bytebase + 2. Navigate to Settings → Service Accounts + 3. Find CI/CD service account + 4. Generate new token + 5. Update CI/CD secrets + + + + Token should be: + - Format: `bb_xxxxxxxxxxxxxxxxxxxxxx` + - No quotes or extra characters + - Properly base64 encoded if required + + **In GitHub Actions:** + ```yaml + env: + BB_TOKEN: ${{ secrets.BYTEBASE_TOKEN }} + # NOT: "${{ secrets.BYTEBASE_TOKEN }}" + ``` + + + +## SQL Review Fails in CI + +**Error:** +``` +SQL review failed: Rule violation - table naming convention +``` + +**Cause:** Migration violates configured SQL review rules. + +**Solutions:** + +**Option 1: Fix the violation** +```sql +-- If error: Table name must be lowercase +-- ❌ Wrong +CREATE TABLE UserProfiles (...); + +-- ✅ Correct +CREATE TABLE user_profiles (...); +``` + +**Option 2: Adjust SQL review policy** + +If rule is too strict: +1. Navigate to SQL Review settings +2. Adjust rule severity (ERROR → WARNING) +3. Or disable rule if not needed + + + Complete reference of 200+ rules + + +**Option 3: Override for specific case** + +Add comment justification in SQL: +```sql +-- bytebase:ignore-rule table.naming.convention +-- Legacy table name required for compatibility +CREATE TABLE UserProfiles (...); +``` + + + Review failures prevent bad changes from reaching production. Fix violations rather than bypassing checks. + + +## CI Timeout + +**Error:** +``` +Error: The operation was canceled. +``` + +**Causes:** +1. Long-running migration exceeds CI timeout +2. Network connectivity issues +3. Bytebase server overloaded + +**Solutions:** + + + + **GitHub Actions:** + ```yaml + jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 30 # Increase from default 360 + steps: + - name: Deploy + timeout-minutes: 20 # Step-level timeout + run: ... + ``` + + **GitLab CI:** + ```yaml + rollout: + timeout: 30m + script: + - bytebase-action rollout ... + ``` + + + + For long-running migrations: + - Break into smaller batches + - Use online schema migration for large tables + - Consider running outside of CI + + + Learn optimization techniques + + + + + Verify Bytebase is responsive: + ```bash + # Test API response time + time curl -f $BB_URL/v1/actuator/info + + # Check Bytebase logs + docker logs bytebase + ``` + + + +## Workflow Dispatch Not Triggering + +**Error:** Manual workflow doesn't start when triggered. + +**Causes:** +1. Wrong branch selected +2. Insufficient permissions +3. Workflow file syntax error + +**Solutions:** + +**Check workflow configuration:** +```yaml +# .github/workflows/deploy.yml +name: Manual Deploy +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to deploy' + required: true + type: choice + options: + - dev + - staging + - production + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Deploy to ${{ inputs.environment }} + run: echo "Deploying to ${{ inputs.environment }}" +``` + +**Verify permissions:** +- Must have write access to repository +- Workflow must be in default branch +- Actions must be enabled for repository + +## Secrets Not Available + +**Error:** +``` +Error: Secret BYTEBASE_TOKEN is not set +``` + +**Solutions:** + +1. **Add secrets to repository:** + - Navigate to Settings → Secrets and variables → Actions + - Click "New repository secret" + - Add required secrets + +2. **Environment-specific secrets:** + ```yaml + jobs: + deploy: + runs-on: ubuntu-latest + environment: production # Use environment secrets + steps: + - name: Deploy + env: + BB_TOKEN: ${{ secrets.BYTEBASE_TOKEN }} + ``` + +3. **Organization secrets:** + - Can be shared across repositories + - Set at organization level + - Select which repositories can access + +--- + +## Next Steps + + + + Step-by-step CI/CD setup guides + + + + Learn about service accounts + + + + CI/CD patterns and strategies + + + + Return to troubleshooting overview + + diff --git a/mintlify/gitops/troubleshooting/overview.mdx b/mintlify/gitops/troubleshooting/overview.mdx new file mode 100644 index 000000000..13ca13bb9 --- /dev/null +++ b/mintlify/gitops/troubleshooting/overview.mdx @@ -0,0 +1,96 @@ +--- +title: Overview +--- + +This troubleshooting guide helps you diagnose and resolve common issues in GitOps workflows with Bytebase. + +## Common Issue Categories + + + + Invalid files, SHA256 mismatches, release not found, plan creation failures + + + + Skipped tasks, pending approvals, SQL errors, permissions, connectivity + + + + Schema qualification, constraints, foreign keys, validation errors + + + + GitHub Actions, authentication, SQL review failures in CI + + + +## Quick Diagnostics + +### Check System Status + +```bash +# Verify Bytebase is accessible +curl -f $BB_URL/v1/actuator/info + +# Test authentication +curl -H "Authorization: Bearer $BB_TOKEN" \ + $BB_URL/v1/actuator/info +``` + +### View Recent Releases + +```bash +# List releases in project +curl -X GET "$BB_URL/v1/projects/my-project/releases" \ + -H "Authorization: Bearer $BB_TOKEN" +``` + +### Check Task Logs + +1. Navigate to the rollout in Bytebase UI +2. Click on the failed task +3. View execution logs for error details + +## Getting Help + +If you've tried these solutions and still need help: + +1. **Check documentation:** + - [API Reference](/integrations/api/overview) + - [Error Codes](/sql-review/error-codes) + - [GitOps Tutorials](/tutorials/gitops-github-workflow) + +2. **Review logs:** + - Task execution logs in Bytebase UI + - CI/CD workflow logs + - Database server logs + +3. **Community support:** + - GitHub Issues: [github.com/bytebase/bytebase/issues](https://github.com/bytebase/bytebase/issues) + - Discord: Join Bytebase community + - Documentation feedback: [docs.bytebase.com](https://docs.bytebase.com) + +4. **Enterprise support:** + - Contact your Bytebase account team + - Submit support ticket via dashboard + - Schedule technical consultation + +## Related Documentation + + + + Understand the complete GitOps workflow + + + + Production-ready patterns + + + + Complete API documentation + + + + SQL review error reference + + diff --git a/mintlify/gitops/troubleshooting/release-and-plan.mdx b/mintlify/gitops/troubleshooting/release-and-plan.mdx new file mode 100644 index 000000000..508c9b63d --- /dev/null +++ b/mintlify/gitops/troubleshooting/release-and-plan.mdx @@ -0,0 +1,228 @@ +--- +title: Release and Plan Issues +--- + +Troubleshoot problems with release creation and plan generation. + +## Release Creation Fails with Invalid File + +**Error:** +``` +Failed to create release: Invalid migration file '002__add_users.sql' +``` + +**Possible Causes:** +1. **Invalid filename format** + - Missing version number + - Invalid characters + - Incorrect suffix + +2. **SQL syntax errors** + - Unparseable SQL + - Database-specific syntax issues + +3. **File encoding issues** + - Non-UTF-8 encoding + - Binary content in SQL file + +**Solutions:** + + + + Ensure filename follows pattern: + ``` + ___.sql + ``` + + Valid examples: + ``` + ✅ 002__add_users.sql + ✅ v1.0.0__init.sql + ✅ 20250120__add_email_dml.sql + ``` + + Invalid examples: + ``` + ❌ add_users.sql (missing version) + ❌ 002-add-users.sql (wrong separator, use __) + ❌ 002__add users.sql (space in filename) + ``` + + + + Test SQL locally: + ```bash + # PostgreSQL + psql -h localhost -U user -d testdb -f 002__add_users.sql + + # MySQL + mysql -h localhost -u user -p testdb < 002__add_users.sql + ``` + + Common syntax issues: + - Missing semicolons + - Incorrect quote types + - Database-specific keywords + + + + Ensure UTF-8 encoding: + ```bash + file 002__add_users.sql + # Should show: UTF-8 Unicode text + + # Convert if needed + iconv -f ISO-8859-1 -t UTF-8 file.sql > file_utf8.sql + ``` + + + +## SHA256 Mismatch Warning + +**Warning:** +``` +Warning: Version 005 already applied with different content +Existing SHA256: abc123... +New SHA256: xyz789... +``` + +**Cause:** Migration file content changed for an existing version. + +**Impact:** File is skipped (idempotent behavior), but indicates inconsistency. + +**Solutions:** + +**Option 1: Accept the difference (if intentional)** +- Document why content differs +- Verify both versions produce same schema +- No action needed (file will be skipped) + +**Option 2: Fix the inconsistency (recommended)** +1. Revert file to original content +2. Create new migration with changes: + ```sql + -- 006__fix_previous_migration.sql + -- Corrects issue from migration 005 + ALTER TABLE ... + ``` + + + Never modify deployed migrations. Always create new migrations for fixes. + + +## Release Not Found + +**Error:** +``` +Release 'projects/my-project/releases/v1.0.0' not found +``` + +**Causes:** +1. Release was deleted +2. Wrong project name +3. Wrong release identifier + +**Solutions:** + +```bash +# List all releases +curl -X GET "$BB_URL/v1/projects/my-project/releases" \ + -H "Authorization: Bearer $BB_TOKEN" + +# Search by digest +curl -X GET "$BB_URL/v1/projects/my-project/releases:search?digest=v1.0.0" \ + -H "Authorization: Bearer $BB_TOKEN" + +# Recreate release if deleted +curl -X POST "$BB_URL/v1/projects/my-project/releases" \ + -H "Authorization: Bearer $BB_TOKEN" \ + -d @release-payload.json +``` + +## Plan Creation Fails - Invalid Target + +**Error:** +``` +Invalid target: instances/prod-mysql/databases/nonexistent +``` + +**Cause:** Specified database doesn't exist or is archived. + +**Solutions:** + + + + ```bash + # List databases in instance + curl -X GET "$BB_URL/v1/instances/prod-mysql/databases" \ + -H "Authorization: Bearer $BB_TOKEN" + ``` + + Check if database: + - Exists with correct name + - Is not archived + - Has correct instance path + + + + If using database group: + ```bash + # List database groups + curl -X GET "$BB_URL/v1/projects/my-project/databaseGroups" \ + -H "Authorization: Bearer $BB_TOKEN" + + # Get specific group members + curl -X GET "$BB_URL/v1/projects/my-project/databaseGroups/prod-fleet" \ + -H "Authorization: Bearer $BB_TOKEN" + ``` + + Verify group: + - Exists + - Contains expected databases + - Has active members + + + +## Plan References Wrong Release + +**Error:** +``` +Release 'projects/my-project/releases/old-release' has no unapplied migrations +``` + +**Cause:** Plan references release where all migrations already applied. + +**Expected Behavior:** This is normal if all migrations are already deployed. + +**Solutions:** + +**If migrations should be unapplied:** +1. Check revision records on target database: + ```bash + curl -X GET "$BB_URL/v1/instances/prod/databases/app_db/revisions" \ + -H "Authorization: Bearer $BB_TOKEN" + ``` + +2. Verify versions in release match: + ```bash + curl -X GET "$BB_URL/v1/projects/my-project/releases/old-release" \ + -H "Authorization: Bearer $BB_TOKEN" + ``` + +**If releasing new changes:** +1. Create new release with new version numbers +2. Update plan to reference new release + +--- + +## Next Steps + + + + Troubleshoot deployment problems + + + + Complete API documentation + + diff --git a/mintlify/gitops/troubleshooting/rollout.mdx b/mintlify/gitops/troubleshooting/rollout.mdx new file mode 100644 index 000000000..1b78a4e36 --- /dev/null +++ b/mintlify/gitops/troubleshooting/rollout.mdx @@ -0,0 +1,274 @@ +--- +title: Rollout Issues +--- + +Troubleshoot problems during rollout execution and task failures. + +## All Tasks Skipped + +**Behavior:** Rollout completes immediately with all tasks skipped. + +**Cause:** All migrations already applied (detected via revision records). + +**Verification:** + +```bash +# Check revisions on target database +curl -X GET "$BB_URL/v1/instances/prod/databases/app_db/revisions" +``` + +**This is normal behavior** indicating idempotent operation. + +**To deploy new changes:** +1. Create migrations with new version numbers +2. Create new release +3. Create new plan/rollout + +## Task Stuck in PENDING_APPROVAL + +**Behavior:** Task waiting for approval indefinitely. + +**Causes:** +1. Rollout policy requires manual approval +2. No authorized users have approved + +**Solutions:** + + + + ```bash + curl -X POST "$BB_URL/v1/projects/my-project/rollouts/rollout-123/stages/stage-1/tasks/task-1:approve" \ + -H "Authorization: Bearer $BB_TOKEN" \ + -d '{"comment": "Approved for deployment"}' + ``` + + + + Verify rollout policy: + ```bash + curl -X GET "$BB_URL/v1/projects/my-project/environments/prod" \ + -H "Authorization: Bearer $BB_TOKEN" + ``` + + Check `rollout_policy` field: + ```json + { + "rollout_policy": { + "automatic": false, + "issue_roles": ["roles/projectOwner", "roles/projectDBA"] + } + } + ``` + + + + Change to automatic (use with caution): + ```bash + curl -X PATCH "$BB_URL/v1/projects/my-project/environments/prod" \ + -H "Authorization: Bearer $BB_TOKEN" \ + -d '{ + "rollout_policy": { + "automatic": true + } + }' + ``` + + + Only enable automatic rollout in non-production environments or with proper safeguards. + + + + +## Task Fails with SQL Error + +**Error in task logs:** +``` +ERROR: syntax error at or near "CRATE" +``` + +**Cause:** SQL syntax error in migration file. + +**Solutions:** + +1. **Identify problematic file:** + - Check task logs for error location + - Identify which migration file failed + +2. **Fix syntax error:** + ```sql + -- Wrong: + CRATE TABLE users (...); + + -- Correct: + CREATE TABLE users (...); + ``` + +3. **Create new release:** + - Don't modify original file if already released + - Create new migration with fix: + ```sql + -- 006__fix_users_table.sql + CREATE TABLE IF NOT EXISTS users (...); + ``` + +4. **Retry or create new rollout:** + ```bash + # Option 1: Retry failed task (if file was corrected before execution) + curl -X POST "$BB_URL/v1/projects/my-project/rollouts/rollout-123/stages/stage-1/tasks:batchRun" \ + -d '{"tasks": ["task-1"]}' + + # Option 2: Create new rollout with corrected release + curl -X POST "$BB_URL/v1/projects/my-project/rollouts" \ + -d '{"plan": "projects/my-project/plans/plan-new"}' + ``` + +## Permission Denied Error + +**Error:** +``` +ERROR: permission denied to create table "users" +``` + +**Cause:** Bytebase database user lacks required permissions. + +**Solutions:** + + + + **For DDL operations (schema changes):** + ```sql + -- PostgreSQL + GRANT CREATE, ALTER, DROP ON DATABASE app_db TO bytebase_user; + GRANT ALL ON SCHEMA public TO bytebase_user; + + -- MySQL + GRANT CREATE, ALTER, DROP, INDEX ON app_db.* TO 'bytebase_user'@'%'; + ``` + + **For DML operations (data changes):** + ```sql + -- PostgreSQL + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO bytebase_user; + + -- MySQL + GRANT SELECT, INSERT, UPDATE, DELETE ON app_db.* TO 'bytebase_user'@'%'; + ``` + + + + If using wrong user in Bytebase: + + 1. Navigate to Instance settings + 2. Update connection username + 3. Save changes + + Or via API: + ```bash + curl -X PATCH "$BB_URL/v1/instances/prod-mysql" \ + -d '{ + "username": "bytebase_admin", + "password": "new_password" + }' + ``` + + + Configure database connections + + + + + Test permissions: + ```sql + -- PostgreSQL: Check grants + SELECT grantee, privilege_type + FROM information_schema.role_table_grants + WHERE grantee = 'bytebase_user'; + + -- MySQL: Show grants + SHOW GRANTS FOR 'bytebase_user'@'%'; + ``` + + + +## Connection Timeout + +**Error:** +``` +Connection timeout after 30 seconds +``` + +**Causes:** +1. Database server down +2. Network connectivity issues +3. Firewall blocking connection +4. Incorrect connection parameters + +**Solutions:** + + + + Check if database is running: + ```bash + # PostgreSQL + pg_isready -h prod-mysql -p 5432 + + # MySQL + mysqladmin -h prod-mysql -u user -p ping + ``` + + + + ```bash + # Test connection + telnet prod-mysql 5432 + + # Or with nc + nc -zv prod-mysql 5432 + + # Test from Bytebase server + docker exec bytebase nc -zv prod-mysql 5432 + ``` + + + + Ensure firewall allows traffic: + - From Bytebase server IP + - To database port (3306/5432/etc.) + - Bidirectional if stateful + + **Cloud providers:** + - AWS: Security Groups + - GCP: Firewall Rules + - Azure: Network Security Groups + + + + Check instance configuration: + ```bash + curl -X GET "$BB_URL/v1/instances/prod-mysql" + ``` + + Verify: + - Host address correct + - Port correct + - SSL/TLS settings if required + + + Database connection setup + + + + +--- + +## Next Steps + + + + Troubleshoot SDL-specific problems + + + + Learn production-ready patterns + + diff --git a/mintlify/gitops/troubleshooting/sdl-issues.mdx b/mintlify/gitops/troubleshooting/sdl-issues.mdx new file mode 100644 index 000000000..fb5c9724a --- /dev/null +++ b/mintlify/gitops/troubleshooting/sdl-issues.mdx @@ -0,0 +1,230 @@ +--- +title: SDL Issues +--- + +Troubleshoot State-Based Workflow (SDL) specific validation errors. + + + SDL is currently supported for PostgreSQL databases only. Validation rules enforce strict conventions to ensure maintainable schema definitions. + + +## Missing Schema Qualification + +**Error:** +``` +Table 'users' must include schema name +``` + +**Cause:** SDL requires all objects to have schema prefix. + +**Solution:** + +```sql +-- ❌ Wrong +CREATE TABLE users (id INTEGER PRIMARY KEY); + +-- ✅ Correct +CREATE TABLE public.users (id INTEGER PRIMARY KEY); +``` + +Apply to all objects: +- Tables: `CREATE TABLE public.users` +- Indexes: `CREATE INDEX idx_name ON public.users(name)` +- Views: `CREATE VIEW public.active_users AS ...` +- Functions: `CREATE FUNCTION public.get_user() ...` + +## Column-Level Constraint + +**Error:** +``` +PRIMARY KEY must be table-level constraint +``` + +**Cause:** SDL requires constraints at table level (except NOT NULL, DEFAULT, GENERATED). + +**Solution:** + +```sql +-- ❌ Wrong - Column-level constraint +CREATE TABLE public.users ( + id INTEGER PRIMARY KEY, + email TEXT UNIQUE +); + +-- ✅ Correct - Table-level constraints +CREATE TABLE public.users ( + id INTEGER, + email TEXT, + CONSTRAINT users_pkey PRIMARY KEY (id), + CONSTRAINT users_email_key UNIQUE (email) +); +``` + +**Allowed at column level:** +```sql +CREATE TABLE public.users ( + id SERIAL, -- ✅ SERIAL allowed + name TEXT NOT NULL, -- ✅ NOT NULL allowed + created_at TIMESTAMP DEFAULT NOW(), -- ✅ DEFAULT allowed + CONSTRAINT users_pkey PRIMARY KEY (id) +); +``` + +## Unnamed Constraint + +**Error:** +``` +All constraints must have explicit names using CONSTRAINT keyword +``` + +**Solution:** + +```sql +-- ❌ Wrong - Unnamed constraints +CREATE TABLE public.orders ( + id INTEGER, + user_id INTEGER, + PRIMARY KEY (id), + FOREIGN KEY (user_id) REFERENCES public.users(id) +); + +-- ✅ Correct - Named constraints +CREATE TABLE public.orders ( + id INTEGER, + user_id INTEGER, + CONSTRAINT orders_pkey PRIMARY KEY (id), + CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES public.users(id) +); +``` + +**Naming conventions:** +- Primary keys: `{table}_pkey` +- Unique constraints: `{table}_{column}_key` +- Foreign keys: `fk_{table}_{referenced_table}` +- Check constraints: `check_{description}` + +## Foreign Key Type Mismatch + +**Error:** +``` +Foreign key column 'user_id' (INTEGER) references 'users.id' (BIGINT) +Types must match exactly +``` + +**Cause:** Column types don't match between foreign key and referenced column. + +**Solution:** + +```sql +-- Ensure types match +CREATE TABLE public.users ( + id BIGINT, + CONSTRAINT users_pkey PRIMARY KEY (id) +); + +CREATE TABLE public.orders ( + id BIGINT, + user_id BIGINT, -- Must match users.id type + CONSTRAINT orders_pkey PRIMARY KEY (id), + CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES public.users(id) +); +``` + +## Foreign Key Missing Schema + +**Error:** +``` +Foreign key reference must include schema name +``` + +**Solution:** + +```sql +-- ❌ Wrong +FOREIGN KEY (user_id) REFERENCES users(id) + +-- ✅ Correct +FOREIGN KEY (user_id) REFERENCES public.users(id) +``` + +## CHECK Constraint References Other Table + +**Error:** +``` +CHECK constraint cannot reference other tables +``` + +**Cause:** CHECK constraints can only reference columns in the same table. + +**Solution:** + +```sql +-- ❌ Wrong - References other table +CREATE TABLE public.orders ( + user_id INTEGER, + CONSTRAINT check_valid_user CHECK (user_id IN (SELECT id FROM public.users)) +); + +-- ✅ Correct - Use foreign key instead +CREATE TABLE public.orders ( + user_id INTEGER, + CONSTRAINT fk_orders_users FOREIGN KEY (user_id) REFERENCES public.users(id) +); +``` + +## Migration Generation Fails + +**Error:** +``` +Failed to generate migration: duplicate object 'users_pkey' +``` + +**Causes:** +1. Duplicate constraint names across tables +2. Duplicate index names +3. Conflicting object names + +**Solution:** + +Ensure all object names are unique: +```sql +-- ❌ Wrong - Duplicate constraint name +CREATE TABLE public.users ( + id INTEGER, + CONSTRAINT pkey PRIMARY KEY (id) +); + +CREATE TABLE public.orders ( + id INTEGER, + CONSTRAINT pkey PRIMARY KEY (id) -- Duplicate name! +); + +-- ✅ Correct - Unique names +CREATE TABLE public.users ( + id INTEGER, + CONSTRAINT users_pkey PRIMARY KEY (id) +); + +CREATE TABLE public.orders ( + id INTEGER, + CONSTRAINT orders_pkey PRIMARY KEY (id) +); +``` + +--- + +## Next Steps + + + + Learn SDL syntax requirements + + + + Understand SDL constraints + + + + Troubleshoot CI/CD problems + +