Working on a dbcopy CLI tool

.vimspector.json

@@ -1,6 +1,27 @@
 {
     "$schema": "https://puremourning.github.io/vimspector/schema/vimspector.schema.json#",
     "configurations": {
+        "dbcopy from and to SQLite3": {
+            "adapter": "vscode-cpptools",
+            "configuration": {
+                "request": "launch",
+                "program": "${workspaceRoot}/out/build/linux-clang-debug/src/tools/dbcopy",
+                "args": [
+                    "--from\\=DRIVER\\=SQLite3\\;DATABASE\\=${workspaceRoot}/Chinook.sqlite",
+                    "--to\\=DRIVER\\=SQLite3\\;DATABASE\\=${workspaceRoot}/output.sqlite"
+                ],
+                "cwd": "${workspaceRoot}",
+                "externalConsole": true,
+                "stopAtEntry": false,
+                "MIMode": "gdb"
+            },
+            "breakpoints": {
+                "exception": {
+                    "caught": "Y",
+                    "uncaught": "Y"
+                }
+            }
+        },
         "CoreTest - SQLite": {
             "adapter": "vscode-cpptools",
             "configuration": {

src/Lightweight/SqlConnection.cpp

@@ -43,7 +43,11 @@ SqlConnection::SqlConnection(std::optional<SqlConnectionString> connectInfo):
     SQLAllocHandle(SQL_HANDLE_DBC, m_hEnv, &m_hDbc);
 
     if (connectInfo.has_value())
-        Connect(std::move(*connectInfo));
+    {
+        auto const success = Connect(std::move(*connectInfo));
+        if (!success)
+            ;
+    }
 }
 
 SqlConnection::SqlConnection(SqlConnection&& other) noexcept:
@@ -189,7 +193,12 @@ bool SqlConnection::Connect(SqlConnectionString sqlConnectionString) noexcept
                                             nullptr,
                                             SQL_DRIVER_NOPROMPT);
     if (!SQL_SUCCEEDED(sqlResult))
+    {
+        auto const errorInfo = SqlErrorInfo::fromConnectionHandle(m_hDbc);
+        SqlLogger::GetLogger().OnError(errorInfo);
+        // throw SqlException(std::move(errorInfo));
         return false;
+    }
 
     sqlResult = SQLSetConnectAttrA(m_hDbc, SQL_ATTR_AUTOCOMMIT, (SQLPOINTER) SQL_AUTOCOMMIT_ON, SQL_IS_UINTEGER);
     if (!SQL_SUCCEEDED(sqlResult))

src/Lightweight/SqlError.hpp

@@ -43,6 +43,11 @@ struct SqlErrorInfo
         return fromHandle(SQL_HANDLE_STMT, hStmt);
     }
 
+    static SqlErrorInfo fromEnvironmentHandle(SQLHSTMT hEnv)
+    {
+        return fromHandle(SQL_HANDLE_ENV, hEnv);
+    }
+
     /// Asserts that the given result is a success code, otherwise throws an exception.
     static void RequireStatementSuccess(SQLRETURN result, SQLHSTMT hStmt, std::string_view message);
 

src/Lightweight/SqlQuery/MigrationPlan.cpp

@@ -13,7 +13,6 @@ std::vector<std::string> SqlMigrationPlan::ToSql() const
     }
     return result;
 }
-
 std::vector<std::string> ToSql(SqlQueryFormatter const& formatter, SqlMigrationPlanElement const& element)
 {
     using namespace std::string_literals;
@@ -38,3 +37,20 @@ std::vector<std::string> ToSql(SqlQueryFormatter const& formatter, SqlMigrationP
         },
         element);
 }
+
+std::vector<std::string> ToSql(std::vector<SqlMigrationPlan> const& plans)
+{
+    std::vector<std::string> result;
+
+    for (auto const& plan: plans)
+    {
+        for (auto const& step: plan.steps)
+        {
+            auto subSteps = ToSql(plan.formatter, step);
+            result.insert(result.end(), subSteps.begin(), subSteps.end());
+        }
+    }
+
+    return result;
+}
+

src/Lightweight/SqlQuery/MigrationPlan.hpp

@@ -357,3 +357,5 @@ struct [[nodiscard]] SqlMigrationPlan
 
     [[nodiscard]] LIGHTWEIGHT_API std::vector<std::string> ToSql() const;
 };
+
+[[nodiscard]] std::vector<std::string> ToSql(SqlQueryFormatter const& formatter, std::vector<SqlMigrationPlan> const& elements);

src/Lightweight/SqlSchema.cpp

@@ -44,13 +44,12 @@ namespace
         return { std::move(first), std::move(second) };
     }
 
-    std::vector<std::string> AllTables(std::string_view database, std::string_view schema)
+    std::vector<std::string> AllTables(SqlStatement& stmt, std::string_view database, std::string_view schema)
     {
         auto const tableType = "TABLE"sv;
         (void) database;
         (void) schema;
 
-        auto stmt = SqlStatement();
         auto sqlResult = SQLTables(stmt.NativeHandle(),
                                    (SQLCHAR*) database.data(),
                                    (SQLSMALLINT) database.size(),
@@ -173,10 +172,9 @@ namespace
 } // namespace
 
 // NOLINTNEXTLINE(readability-function-cognitive-complexity)
-void ReadAllTables(std::string_view database, std::string_view schema, EventHandler& eventHandler)
+void ReadAllTables(SqlStatement& stmt, std::string_view database, std::string_view schema, EventHandler& eventHandler)
 {
-    auto stmt = SqlStatement {};
-    auto const tableNames = AllTables(database, schema);
+    auto const tableNames = AllTables(stmt, database, schema);
 
     eventHandler.OnTables(tableNames);
 
@@ -206,7 +204,7 @@ void ReadAllTables(std::string_view database, std::string_view schema, EventHand
         for (auto const& foreignKey: incomingForeignKeys)
             eventHandler.OnExternalForeignKey(foreignKey);
 
-        auto columnStmt = SqlStatement();
+        auto columnStmt = SqlStatement { stmt.Connection() };
         auto const sqlResult = SQLColumns(columnStmt.NativeHandle(),
                                           (SQLCHAR*) database.data(),
                                           (SQLSMALLINT) database.size(),
@@ -274,7 +272,6 @@ void ReadAllTables(std::string_view database, std::string_view schema, EventHand
 
             // accumulated properties
             column.isPrimaryKey = std::ranges::contains(primaryKeys, column.name);
-            // column.isForeignKey = ...;
             column.isForeignKey = std::ranges::any_of(foreignKeys, [&column](ForeignKeyConstraint const& fk) {
                 return std::ranges::contains(fk.foreignKey.columns, column.name);
             });
@@ -301,7 +298,10 @@ std::string ToLowerCase(std::string_view str)
     return result;
 }
 
-TableList ReadAllTables(std::string_view database, std::string_view schema, ReadAllTablesCallback callback)
+TableList ReadAllTables(SqlStatement& stmt,
+                        std::string_view database,
+                        std::string_view schema,
+                        ReadAllTablesCallback callback)
 {
     TableList tables;
     struct EventHandler: public SqlSchema::EventHandler
@@ -353,7 +353,7 @@ TableList ReadAllTables(std::string_view database, std::string_view schema, Read
             tables.back().externalForeignKeys.emplace_back(foreignKeyConstraint);
         }
     } eventHandler { tables, std::move(callback) };
-    ReadAllTables(database, schema, eventHandler);
+    ReadAllTables(stmt, database, schema, eventHandler);
 
     std::map<std::string, std::string> tableNameCaseMap;
     for (auto const& table: tables)
@@ -388,4 +388,45 @@ std::vector<ForeignKeyConstraint> AllForeignKeysFrom(SqlStatement& stmt, FullyQu
     return AllForeignKeys(stmt, FullyQualifiedTableName {}, table);
 }
 
+SqlCreateTablePlan MakeCreateTablePlan(Table const& tableDescription)
+{
+    auto plan = SqlCreateTablePlan {}; // TODO(pr)
+
+    plan.tableName = tableDescription.name;
+
+    for (auto const& columnDescription: tableDescription.columns)
+    {
+        auto columnDecl = SqlColumnDeclaration {
+            .name = columnDescription.name,
+            .type = columnDescription.type,
+            .primaryKey = [&] {
+                if (columnDescription.isAutoIncrement)
+                    return SqlPrimaryKeyType::AUTO_INCREMENT;
+
+                if (columnDescription.isPrimaryKey)
+                    return SqlPrimaryKeyType::MANUAL;
+
+                 return SqlPrimaryKeyType::NONE;
+            }(),
+            .foreignKey = {}, // TODO(pr) foreign keys
+            .required = !columnDescription.isNullable,
+            .unique = columnDescription.isUnique,
+        };
+
+        plan.columns.emplace_back(std::move(columnDecl));
+    }
+
+    return plan;
+}
+
+std::vector<SqlCreateTablePlan> MakeCreateTablePlan(TableList const& tableDescriptions)
+{
+    auto result = std::vector<SqlCreateTablePlan>();
+
+    for (auto const& tableDescription: tableDescriptions)
+        result.emplace_back(MakeCreateTablePlan(tableDescription));
+
+    return result;
+}
+
 } // namespace SqlSchema

src/Lightweight/SqlSchema.hpp

@@ -130,7 +130,10 @@ class EventHandler
 };
 
 /// Reads all tables in the given database and schema and calls the event handler for each table.
-LIGHTWEIGHT_API void ReadAllTables(std::string_view database, std::string_view schema, EventHandler& eventHandler);
+LIGHTWEIGHT_API void ReadAllTables(SqlStatement& stmt,
+                                   std::string_view database,
+                                   std::string_view schema,
+                                   EventHandler& eventHandler);
 
 /// Holds the definition of a table in a SQL database as read from the database schema.
 struct Table
@@ -159,7 +162,8 @@ using TableList = std::vector<Table>;
 using ReadAllTablesCallback = std::function<void(std::string_view /*tableName*/, size_t /*current*/, size_t /*total*/)>;
 
 /// Retrieves all tables in the given @p database and @p schema.
-LIGHTWEIGHT_API TableList ReadAllTables(std::string_view database,
+LIGHTWEIGHT_API TableList ReadAllTables(SqlStatement& stmt,
+                                        std::string_view database,
                                         std::string_view schema = {},
                                         ReadAllTablesCallback callback = {});
 
@@ -170,6 +174,12 @@ LIGHTWEIGHT_API std::vector<ForeignKeyConstraint> AllForeignKeysTo(SqlStatement&
 LIGHTWEIGHT_API std::vector<ForeignKeyConstraint> AllForeignKeysFrom(SqlStatement& stmt,
                                                                      FullyQualifiedTableName const& table);
 
+/// Creats an SQL CREATE TABLE plan for the given table description.
+LIGHTWEIGHT_API SqlCreateTablePlan MakeCreateTablePlan(Table const& tableDescription);
+
+/// Creates an SQL CREATE TABLE plan for all the given table descriptions.
+LIGHTWEIGHT_API std::vector<SqlCreateTablePlan> MakeCreateTablePlan(TableList const& tableDescriptions);
+
 } // namespace SqlSchema
 
 template <>

src/tools/CMakeLists.txt

@@ -3,3 +3,8 @@ add_executable(ddl2cpp ddl2cpp.cpp)
 target_link_libraries(ddl2cpp PRIVATE Lightweight::Lightweight yaml-cpp)
 target_compile_features(ddl2cpp PUBLIC cxx_std_23)
 install(TARGETS ddl2cpp DESTINATION bin)
+
+add_executable(dbcopy dbcopy.cpp)
+target_link_libraries(dbcopy PRIVATE Lightweight::Lightweight)
+target_compile_features(dbcopy PUBLIC cxx_std_23)
+install(TARGETS dbcopy DESTINATION bin)

src/tools/dbcopy.cpp

@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include "utils.hpp"
+
+#include <Lightweight/Lightweight.hpp>
+
+#include <print>
+
+using namespace std::string_view_literals;
+
+struct Configuration
+{
+    std::string from;
+    std::string schema;
+    std::string to;
+    bool help = false;
+    bool quiet = false;
+};
+
+namespace
+{
+
+void PrintHelp()
+{
+    std::println("dbcopy [--help] [--quiet] --from=DSN [--schema=NAME] --to=DSN");
+    std::println();
+}
+
+std::vector<std::string> ToSql(SqlQueryFormatter const& formatter,
+                               std::vector<SqlCreateTablePlan> const& createTableMigrationPlan)
+{
+    auto result = std::vector<std::string> {};
+    for (SqlCreateTablePlan const& createTablePlan: createTableMigrationPlan)
+        std::ranges::move(ToSql(formatter, SqlMigrationPlanElement { createTablePlan }), std::back_inserter(result));
+    return result;
+}
+
+} // end namespace
+
+int main(int argc, char const* argv[])
+{
+    auto config = Configuration {};
+    if (!ParseCommandLineArguments(&config, argc, argv) && !config.help)
+    {
+        std::println("Error: Invalid command line arguments");
+        return EXIT_FAILURE;
+    }
+
+    if (config.help)
+    {
+        PrintHelp();
+        return EXIT_SUCCESS;
+    }
+
+    auto const sourceConnectionString = SqlConnectionString { config.from };
+    auto sourceConnection = SqlConnection { sourceConnectionString };
+
+    SqlStatement sourceStmt { sourceConnection };
+    SqlSchema::TableList tableSchemas =
+        SqlSchema::ReadAllTables(sourceStmt, {}, config.schema, [&](std::string_view table, size_t current, size_t total) {
+            if (!config.quiet)
+                std::println("Processing table {}/{}: {}", current, total, table);
+        });
+
+    if (!config.quiet)
+        std::println("Read {} tables from source database", tableSchemas.size());
+
+    auto const createTableMigrationPlan = MakeCreateTablePlan(tableSchemas);
+
+    if (config.to.empty() || config.to == "-"sv)
+    {
+        // Dump all SQL queries to stdout for now
+        std::println("SQL queries for target database:");
+        auto const sqlTargetServerType = SqlServerType::SQLITE; // TODO(pr) determine automatically
+        auto const& formatter = *SqlQueryFormatter::Get(sqlTargetServerType);
+        auto const sqlQueries = ToSql(formatter, createTableMigrationPlan);
+        for (auto const& sqlQuery: sqlQueries)
+            std::println("{}\n", sqlQuery);
+    }
+    else
+    {
+        // Implement writing to target database
+        auto targetConnection = SqlConnection { SqlConnectionString { config.to } };
+        auto const sqlQueries = ToSql(targetConnection.QueryFormatter(), createTableMigrationPlan);
+
+        // TODO(pr): Execute the SQL queries on the target connection
+    }
+
+    // TODO(pr): Copy data from source to target database
+
+    return EXIT_SUCCESS;
+}

src/tools/ddl2cpp.cpp

@@ -1105,8 +1105,9 @@ int main(int argc, char const* argv[])
     PrintInfo(config);
 
     std::vector<SqlSchema::Table> const tables = TimedExecution("Reading all tables", [&] {
+        SqlStatement stmt;
         return SqlSchema::ReadAllTables(
-            config.database, config.schema, [](std::string_view tableName, size_t current, size_t total) {
+            stmt, config.database, config.schema, [](std::string_view tableName, size_t current, size_t total) {
                 std::print("\r\033[K {:>3}% [{}/{}] Reading table schema {}",
                            static_cast<int>((current * 100) / total),
                            current,