Merge pull request #70 from objectstack-ai/copilot/sync-sql-database-objects

Author: huangyiirene

packages/drivers/sql/src/index.ts

@@ -387,6 +387,12 @@ export class SqlDriver implements Driver {
             case 'file':
             case 'avatar':
             case 'location': col = table.json(name); break;
+            case 'lookup': 
+                col = table.string(name);
+                if (field.reference_to) {
+                    table.foreign(name).references('id').inTable(field.reference_to);
+                }
+                break;
             case 'summary': col = table.float(name); break; // Stored calculation result
             case 'auto_number': col = table.string(name); break; // Generated string
             case 'formula': return; // Virtual field, do not create column
@@ -532,6 +538,17 @@ export class SqlDriver implements Driver {
             const columns = await this.introspectColumns(tableName);
             const foreignKeys = await this.introspectForeignKeys(tableName);
             const primaryKeys = await this.introspectPrimaryKeys(tableName);
+            const uniqueConstraints = await this.introspectUniqueConstraints(tableName);
+            
+            // Update columns with primary key and unique information
+            for (const col of columns) {
+                if (primaryKeys.includes(col.name)) {
+                    col.isPrimary = true;
+                }
+                if (uniqueConstraints.includes(col.name)) {
+                    col.isUnique = true;
+                }
+            }
 
             tables[tableName] = {
                 name: tableName,
@@ -730,6 +747,72 @@ export class SqlDriver implements Driver {
         return primaryKeys;
     }
 
+    /**
+     * Get unique constraint information for a specific table.
+     */
+    private async introspectUniqueConstraints(tableName: string): Promise<string[]> {
+        const uniqueColumns: string[] = [];
+        
+        try {
+            if (this.config.client === 'pg' || this.config.client === 'postgresql') {
+                const result = await this.knex.raw(`
+                    SELECT c.column_name
+                    FROM information_schema.table_constraints tc
+                    JOIN information_schema.constraint_column_usage AS ccu 
+                        ON tc.constraint_schema = ccu.constraint_schema
+                        AND tc.constraint_name = ccu.constraint_name
+                    WHERE tc.constraint_type = 'UNIQUE' 
+                        AND tc.table_name = ?
+                `, [tableName]);
+                
+                for (const row of result.rows) {
+                    uniqueColumns.push(row.column_name);
+                }
+            } else if (this.config.client === 'mysql' || this.config.client === 'mysql2') {
+                const result = await this.knex.raw(`
+                    SELECT COLUMN_NAME
+                    FROM information_schema.TABLE_CONSTRAINTS tc
+                    JOIN information_schema.KEY_COLUMN_USAGE kcu
+                        USING (CONSTRAINT_NAME, TABLE_SCHEMA, TABLE_NAME)
+                    WHERE CONSTRAINT_TYPE = 'UNIQUE'
+                        AND TABLE_SCHEMA = DATABASE()
+                        AND TABLE_NAME = ?
+                `, [tableName]);
+                
+                for (const row of result[0]) {
+                    uniqueColumns.push(row.COLUMN_NAME);
+                }
+            } else if (this.config.client === 'sqlite3') {
+                const safeTableName = tableName.replace(/[^a-zA-Z0-9_]/g, '');
+                
+                // Validate table exists
+                const tablesResult = await this.knex.raw("SELECT name FROM sqlite_master WHERE type = 'table'");
+                const tableNames = Array.isArray(tablesResult) ? tablesResult.map((row: any) => row.name) : [];
+                
+                if (!tableNames.includes(safeTableName)) {
+                    return uniqueColumns;
+                }
+
+                const indexes = await this.knex.raw(`PRAGMA index_list(${safeTableName})`);
+                
+                for (const idx of indexes) {
+                     // Check if unique
+                    if (idx.unique === 1) {
+                         const info = await this.knex.raw(`PRAGMA index_info(${idx.name})`);
+                         // Only handle single column unique constraints for now
+                         if (info.length === 1) {
+                             uniqueColumns.push(info[0].name);
+                         }
+                    }
+                }
+            }
+        } catch (error) {
+            console.warn('Could not introspect unique constraints for a table:', error);
+        }
+        
+        return uniqueColumns;
+    }
+
     async disconnect() {
         await this.knex.destroy();
     }

packages/foundation/core/src/app.ts

@@ -197,6 +197,15 @@ export class ObjectQL implements IObjectQL {
         return objects;
     }
 
+    async close() {
+        for (const [name, driver] of Object.entries(this.datasources)) {
+            if (driver.disconnect) {
+                console.log(`Closing driver '${name}'...`);
+                await driver.disconnect();
+            }
+        }
+    }
+
     async init() {
         // 0. Init Plugins (This allows plugins to register custom loaders)
         for (const plugin of this.pluginsList) {

packages/foundation/types/src/app.ts

@@ -10,6 +10,7 @@ export interface IObjectQL {
     getConfigs(): Record<string, ObjectConfig>;
     datasource(name: string): Driver;
     init(): Promise<void>;
+    close?(): Promise<void>;
     removePackage(name: string): void;
     metadata: MetadataRegistry; 
 

packages/tools/cli/README.md

@@ -356,6 +356,79 @@ objectql migrate status --config ./config/objectql.config.ts
 - `-c, --config <path>` - Path to objectql.config.ts/js
 - `-d, --dir <path>` - Migrations directory [default: "./migrations"]
 
+#### `sync`
+
+Introspect an existing SQL database and generate ObjectQL `.object.yml` files from the database schema. This is useful for:
+- Connecting to an existing/legacy database
+- Reverse-engineering database schema to ObjectQL metadata
+- Bootstrapping a new ObjectQL project from an existing database
+
+```bash
+# Sync all tables from the database
+objectql sync
+
+# Sync specific tables only
+objectql sync --tables users posts comments
+
+# Custom output directory
+objectql sync --output ./src/metadata/objects
+
+# Overwrite existing files
+objectql sync --force
+
+# With custom config file
+objectql sync --config ./config/objectql.config.ts
+```
+
+**Options:**
+- `-c, --config <path>` - Path to objectql.config.ts/js
+- `-o, --output <path>` - Output directory for .object.yml files [default: "./src/objects"]
+- `-t, --tables <tables...>` - Specific tables to sync (default: all tables)
+- `-f, --force` - Overwrite existing .object.yml files
+
+**Features:**
+- Automatically detects table structure (columns, data types, constraints)
+- Maps SQL types to appropriate ObjectQL field types
+- Identifies foreign keys and converts them to `lookup` relationships
+- Generates human-readable labels from table/column names
+- Preserves field constraints (required, unique, maxLength)
+- Skips system fields (id, created_at, updated_at) as they're automatic in ObjectQL
+
+**Example:**
+
+Given a database with this table structure:
+```sql
+CREATE TABLE users (
+    id VARCHAR PRIMARY KEY,
+    username VARCHAR UNIQUE NOT NULL,
+    email VARCHAR NOT NULL,
+    is_active BOOLEAN DEFAULT true,
+    created_at TIMESTAMP,
+    updated_at TIMESTAMP
+);
+```
+
+Running `objectql sync` generates:
+```yaml
+# users.object.yml
+name: users
+label: Users
+fields:
+  username:
+    type: text
+    label: Username
+    required: true
+    unique: true
+  email:
+    type: text
+    label: Email
+    required: true
+  is_active:
+    type: boolean
+    label: Is Active
+    defaultValue: true
+```
+
 ---
 
 ### Development Tools

packages/tools/cli/USAGE_EXAMPLES.md

@@ -517,6 +517,153 @@ export async function up(app: ObjectQL) {
 }
 ```
 
+### Syncing from Existing Database
+
+The `sync` command introspects an existing SQL database and generates ObjectQL `.object.yml` files.
+
+#### Basic Sync
+
+```bash
+# Sync all tables from the database
+objectql sync
+
+# Output: src/objects/users.object.yml, src/objects/posts.object.yml, etc.
+```
+
+#### Selective Table Sync
+
+```bash
+# Sync only specific tables
+objectql sync --tables users posts comments
+
+# Custom output directory
+objectql sync --output ./src/metadata/objects
+```
+
+#### Overwriting Existing Files
+
+```bash
+# Force overwrite of existing .object.yml files
+objectql sync --force
+```
+
+#### Example Workflow: Connecting to Legacy Database
+
+```bash
+# 1. Create config file pointing to existing database
+cat > objectql.config.ts << 'EOF'
+import { ObjectQL } from '@objectql/core';
+import { SqlDriver } from '@objectql/driver-sql';
+
+const driver = new SqlDriver({
+    client: 'postgresql',
+    connection: {
+        host: 'localhost',
+        database: 'legacy_db',
+        user: 'postgres',
+        password: 'password'
+    }
+});
+
+export default new ObjectQL({
+    datasources: { default: driver }
+});
+EOF
+
+# 2. Introspect and generate object definitions
+objectql sync --output ./src/objects
+
+# 3. Review generated files
+ls -la ./src/objects/
+
+# Output:
+# users.object.yml
+# products.object.yml
+# orders.object.yml
+# order_items.object.yml
+
+# 4. Inspect a generated file
+cat ./src/objects/users.object.yml
+```
+
+**Generated Output Example:**
+
+```yaml
+# users.object.yml
+name: users
+label: Users
+fields:
+  username:
+    type: text
+    label: Username
+    required: true
+    unique: true
+  email:
+    type: text
+    label: Email
+    required: true
+  first_name:
+    type: text
+    label: First Name
+  last_name:
+    type: text
+    label: Last Name
+  is_active:
+    type: boolean
+    label: Is Active
+    defaultValue: true
+  role_id:
+    type: lookup
+    label: Role Id
+    reference_to: roles
+```
+
+**Type Mapping:**
+
+The sync command automatically maps SQL types to ObjectQL field types:
+
+| SQL Type | ObjectQL Type |
+|----------|---------------|
+| INT, INTEGER, BIGINT, SERIAL | number |
+| FLOAT, DOUBLE, DECIMAL, NUMERIC | number |
+| BOOLEAN, BIT | boolean |
+| VARCHAR, CHAR | text |
+| TEXT, CLOB, LONGTEXT | textarea |
+| TIMESTAMP, DATETIME | datetime |
+| DATE | date |
+| TIME | time |
+| JSON, JSONB | object |
+| BLOB, BINARY, BYTEA | file |
+
+**Foreign Key Detection:**
+
+Foreign keys are automatically detected and converted to `lookup` fields:
+
+```sql
+-- Database Schema
+CREATE TABLE posts (
+    id VARCHAR PRIMARY KEY,
+    title VARCHAR NOT NULL,
+    author_id VARCHAR REFERENCES users(id),
+    ...
+);
+```
+
+```yaml
+# Generated posts.object.yml
+name: posts
+label: Posts
+fields:
+  title:
+    type: text
+    label: Title
+    required: true
+  author_id:
+    type: lookup
+    label: Author Id
+    reference_to: users
+```
+
 ---
 
 ## Development Tools