Update sqlpage and sqlx dependencies, refactor database handling to use SupportedDatabase enum for improved database type management and placeholder generation.

Author: lovasoa

Cargo.lock

@@ -1,19 +1,14 @@
 [package]
 name = "sqlpage"
-version = "0.37.1"
+version = "0.38.0-beta.1"
 edition = "2021"
 description = "Build data user interfaces entirely in SQL. A web server that takes .sql files and formats the query result using pre-made configurable professional-looking components."
 keywords = ["web", "sql", "framework"]
 license = "MIT"
 homepage = "https://sql-page.com/"
 repository = "https://github.com/sqlpage/SQLPage"
 documentation = "https://docs.rs/sqlpage"
-include = [
-    "/src",
-    "/README.md",
-    "/build.rs",
-    "/sqlpage",
-]
+include = ["/src", "/README.md", "/build.rs", "/sqlpage"]
 
 [profile.superoptimized]
 inherits = "release"
@@ -23,7 +18,7 @@ panic = "abort"
 codegen-units = 2
 
 [dependencies]
-sqlx = { package = "sqlx-oldapi", version = "0.6.49-beta", default-features = false, features = [
+sqlx = { package = "sqlx-oldapi", version = "0.6.49-beta.2", default-features = false, features = [
     "any",
     "runtime-tokio-rustls",
     "migrate",
@@ -50,7 +45,10 @@ anyhow = "1"
 serde = "1"
 serde_json = { version = "1.0.82", features = ["preserve_order", "raw_value"] }
 lambda-web = { version = "0.2.1", features = ["actix4"], optional = true }
-sqlparser = { version = "0.59.0", default-features = false, features = ["std", "visitor",] }
+sqlparser = { version = "0.59.0", default-features = false, features = [
+    "std",
+    "visitor",
+] }
 async-stream = "0.3"
 async-trait = "0.1.61"
 async-recursion = "1.0.0"

Cargo.toml

@@ -3,7 +3,8 @@ use crate::webserver::{make_placeholder, Database};
 use crate::{AppState, TEMPLATES_DIR};
 use anyhow::Context;
 use chrono::{DateTime, Utc};
-use sqlx::any::{AnyKind, AnyStatement, AnyTypeInfo};
+use sqlx::any::{AnyStatement, AnyTypeInfo};
+use crate::webserver::database::SupportedDatabase;
 use sqlx::postgres::types::PgTimeTz;
 use sqlx::{Postgres, Statement, Type};
 use std::fmt::Write;
@@ -27,7 +28,7 @@ impl FileSystem {
                         You can host sql files directly in your database by creating the following table: \n\
                         {} \n\
                         The error while trying to use the database file system is: {e:#}",
-                        DbFsQueries::get_create_table_sql(db.connection.any_kind())
+                        DbFsQueries::get_create_table_sql(db.database_type)
                     );
                     None
                 }
@@ -206,32 +207,32 @@ pub struct DbFsQueries {
 
 impl DbFsQueries {
     #[must_use]
-    pub fn get_create_table_sql(db_kind: AnyKind) -> &'static str {
-        match db_kind {
-            AnyKind::Mssql => "CREATE TABLE sqlpage_files(path NVARCHAR(255) NOT NULL PRIMARY KEY, contents VARBINARY(MAX), last_modified DATETIME2(3) NOT NULL DEFAULT CURRENT_TIMESTAMP);",
-            AnyKind::Postgres => "CREATE TABLE IF NOT EXISTS sqlpage_files(path VARCHAR(255) NOT NULL PRIMARY KEY, contents BYTEA, last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP);",
+    pub fn get_create_table_sql(dbms: SupportedDatabase) -> &'static str {
+        match dbms {
+            SupportedDatabase::Mssql => "CREATE TABLE sqlpage_files(path NVARCHAR(255) NOT NULL PRIMARY KEY, contents VARBINARY(MAX), last_modified DATETIME2(3) NOT NULL DEFAULT CURRENT_TIMESTAMP);",
+            SupportedDatabase::Postgres => "CREATE TABLE IF NOT EXISTS sqlpage_files(path VARCHAR(255) NOT NULL PRIMARY KEY, contents BYTEA, last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP);",
             _ => "CREATE TABLE IF NOT EXISTS sqlpage_files(path VARCHAR(255) NOT NULL PRIMARY KEY, contents BLOB, last_modified TIMESTAMP DEFAULT CURRENT_TIMESTAMP);",
         }
     }
 
     async fn init(db: &Database) -> anyhow::Result<Self> {
         log::debug!("Initializing database filesystem queries");
-        let db_kind = db.connection.any_kind();
+        let dbms = db.database_type;
         Ok(Self {
-            was_modified: Self::make_was_modified_query(db, db_kind).await?,
-            read_file: Self::make_read_file_query(db, db_kind).await?,
-            exists: Self::make_exists_query(db, db_kind).await?,
+            was_modified: Self::make_was_modified_query(db, dbms).await?,
+            read_file: Self::make_read_file_query(db, dbms).await?,
+            exists: Self::make_exists_query(db, dbms).await?,
         })
     }
 
     async fn make_was_modified_query(
         db: &Database,
-        db_kind: AnyKind,
+        dbms: SupportedDatabase,
     ) -> anyhow::Result<AnyStatement<'static>> {
         let was_modified_query = format!(
             "SELECT 1 from sqlpage_files WHERE last_modified >= {} AND path = {}",
-            make_placeholder(db_kind, 1),
-            make_placeholder(db_kind, 2)
+            make_placeholder(dbms, 1),
+            make_placeholder(dbms, 2)
         );
         let param_types: &[AnyTypeInfo; 2] = &[
             PgTimeTz::type_info().into(),
@@ -243,11 +244,11 @@ impl DbFsQueries {
 
     async fn make_read_file_query(
         db: &Database,
-        db_kind: AnyKind,
+        dbms: SupportedDatabase,
     ) -> anyhow::Result<AnyStatement<'static>> {
         let read_file_query = format!(
             "SELECT contents from sqlpage_files WHERE path = {}",
-            make_placeholder(db_kind, 1),
+            make_placeholder(dbms, 1),
         );
         let param_types: &[AnyTypeInfo; 1] = &[<str as Type<Postgres>>::type_info().into()];
         log::debug!("Preparing the database filesystem read_file_query: {read_file_query}");
@@ -256,11 +257,11 @@ impl DbFsQueries {
 
     async fn make_exists_query(
         db: &Database,
-        db_kind: AnyKind,
+        dbms: SupportedDatabase,
     ) -> anyhow::Result<AnyStatement<'static>> {
         let exists_query = format!(
             "SELECT 1 from sqlpage_files WHERE path = {}",
-            make_placeholder(db_kind, 1),
+            make_placeholder(dbms, 1),
         );
         let param_types: &[AnyTypeInfo; 1] = &[<str as Type<Postgres>>::type_info().into()];
         db.prepare_with(&exists_query, param_types).await
@@ -356,11 +357,11 @@ async fn test_sql_file_read_utf8() -> anyhow::Result<()> {
         .execute(format!("DROP TABLE IF EXISTS sqlpage_files; {create_table_sql}").as_str())
         .await?;
 
-    let db_kind = state.db.connection.any_kind();
+    let dbms = state.db.database_type;
     let insert_sql = format!(
         "INSERT INTO sqlpage_files(path, contents) VALUES ({}, {})",
-        make_placeholder(db_kind, 1),
-        make_placeholder(db_kind, 2)
+        make_placeholder(dbms, 1),
+        make_placeholder(dbms, 2)
     );
     sqlx::query(&insert_sql)
         .bind("unit test file.txt")

src/filesystem.rs

@@ -1,14 +1,16 @@
 use std::{mem::take, time::Duration};
 
 use super::Database;
-use crate::{app_config::AppConfig, ON_CONNECT_FILE, ON_RESET_FILE};
+use crate::{
+    app_config::AppConfig, webserver::database::SupportedDatabase, ON_CONNECT_FILE, ON_RESET_FILE,
+};
 use anyhow::Context;
 use futures_util::future::BoxFuture;
 use sqlx::{
-    any::{Any, AnyConnectOptions, AnyKind},
+    any::{Any, AnyConnectOptions},
     pool::PoolOptions,
     sqlite::{Function, SqliteFunctionCtx},
-    ConnectOptions, Executor,
+    ConnectOptions, Connection, Executor,
 };
 
 impl Database {
@@ -33,8 +35,11 @@ impl Database {
         set_custom_connect_options(&mut connect_options, config);
         log::debug!("Connecting to database: {database_url}");
         let mut retries = config.database_connection_retries;
-        let connection = loop {
-            match Self::create_pool_options(config, connect_options.kind())
+        // Try to determine database type from connection string first
+        let kind_str = format!("{:?}", connect_options.kind()).to_lowercase();
+        let database_type = SupportedDatabase::from_dbms_name(&kind_str);
+        let pool = loop {
+            match Self::create_pool_options(config, database_type)
                 .connect_with(connect_options.clone())
                 .await
             {
@@ -50,44 +55,48 @@ impl Database {
                 }
             }
         };
-        log::debug!("Initialized database pool: {connection:#?}");
-        Ok(Database { connection })
+        let _dbms_name: String = pool.acquire().await?.dbms_name().await?;
+        log::debug!("Initialized database pool: {pool:#?}");
+        Ok(Database {
+            connection: pool,
+            database_type,
+        })
     }
 
-    fn create_pool_options(config: &AppConfig, db_kind: AnyKind) -> PoolOptions<Any> {
+    fn create_pool_options(config: &AppConfig, dbms: SupportedDatabase) -> PoolOptions<Any> {
         let mut pool_options = PoolOptions::new()
             .max_connections(if let Some(max) = config.max_database_pool_connections {
                 max
             } else {
                 // Different databases have a different number of max concurrent connections allowed by default
-                match db_kind {
-                    AnyKind::Postgres | AnyKind::Odbc => 50, // Default to PostgreSQL-like limits for ODBC
-                    AnyKind::MySql => 75,
-                    AnyKind::Sqlite => {
+                match dbms {
+                    SupportedDatabase::Postgres | SupportedDatabase::Generic => 50, // Default to PostgreSQL-like limits for Generic
+                    SupportedDatabase::MySql => 75,
+                    SupportedDatabase::Sqlite => {
                         if config.database_url.contains(":memory:") {
                             128
                         } else {
                             16
                         }
                     }
-                    AnyKind::Mssql => 100,
+                    SupportedDatabase::Mssql => 100,
                 }
             })
             .idle_timeout(
                 config
                     .database_connection_idle_timeout_seconds
                     .map(Duration::from_secs_f64)
-                    .or_else(|| match db_kind {
-                        AnyKind::Sqlite => None,
+                    .or_else(|| match dbms {
+                        SupportedDatabase::Sqlite => None,
                         _ => Some(Duration::from_secs(30 * 60)),
                     }),
             )
             .max_lifetime(
                 config
                     .database_connection_max_lifetime_seconds
                     .map(Duration::from_secs_f64)
-                    .or_else(|| match db_kind {
-                        AnyKind::Sqlite => None,
+                    .or_else(|| match dbms {
+                        SupportedDatabase::Sqlite => None,
                         _ => Some(Duration::from_secs(60 * 60)),
                     }),
             )

src/webserver/database/connect.rs

@@ -6,9 +6,10 @@ use sqlparser::ast::{
     CopyLegacyCsvOption, CopyLegacyOption, CopyOption, CopySource, CopyTarget, Statement,
 };
 use sqlx::{
-    any::{AnyArguments, AnyConnectionKind, AnyKind},
+    any::{AnyArguments, AnyConnectionKind},
     AnyConnection, Arguments, Executor, PgConnection,
 };
+use crate::webserver::database::SupportedDatabase;
 use tokio::io::AsyncRead;
 
 use crate::webserver::http_request_info::RequestInfo;
@@ -143,6 +144,7 @@ pub(super) fn extract_csv_copy_statement(stmt: &mut Statement) -> Option<CsvImpo
 
 pub(super) async fn run_csv_import(
     db: &mut AnyConnection,
+    dbms: SupportedDatabase,
     csv_import: &CsvImport,
     request: &RequestInfo,
 ) -> anyhow::Result<()> {
@@ -173,7 +175,7 @@ pub(super) async fn run_csv_import(
         AnyConnectionKind::Postgres(pg_connection) => {
             run_csv_import_postgres(pg_connection, csv_import, buffered).await
         }
-        _ => run_csv_import_insert(db, csv_import, buffered).await,
+        _ => run_csv_import_insert(db, dbms, csv_import, buffered).await,
     }
     .with_context(|| {
         let table_name = &csv_import.table_name;
@@ -216,10 +218,11 @@ async fn run_csv_import_postgres(
 
 async fn run_csv_import_insert(
     db: &mut AnyConnection,
+    dbms: SupportedDatabase,
     csv_import: &CsvImport,
     file: impl AsyncRead + Unpin + Send,
 ) -> anyhow::Result<()> {
-    let insert_stmt = create_insert_stmt(db.kind(), csv_import);
+    let insert_stmt = create_insert_stmt(dbms, csv_import);
     log::debug!("CSV data insert statement: {insert_stmt}");
     let mut reader = make_csv_reader(csv_import, file);
     let col_idxs = compute_column_indices(&mut reader, csv_import).await?;
@@ -256,13 +259,13 @@ async fn compute_column_indices<R: AsyncRead + Unpin + Send>(
     Ok(col_idxs)
 }
 
-fn create_insert_stmt(kind: AnyKind, csv_import: &CsvImport) -> String {
+fn create_insert_stmt(dbms: SupportedDatabase, csv_import: &CsvImport) -> String {
     let columns = csv_import.columns.join(", ");
     let placeholders = csv_import
         .columns
         .iter()
         .enumerate()
-        .map(|(i, _)| make_placeholder(kind, i + 1))
+        .map(|(i, _)| make_placeholder(dbms, i + 1))
         .fold(String::new(), |mut acc, f| {
             if !acc.is_empty() {
                 acc.push_str(", ");
@@ -328,7 +331,7 @@ fn test_make_statement() {
         escape: None,
         uploaded_file: "my_file.csv".into(),
     };
-    let insert_stmt = create_insert_stmt(AnyKind::Postgres, &csv_import);
+    let insert_stmt = create_insert_stmt(SupportedDatabase::Postgres, &csv_import);
     assert_eq!(
         insert_stmt,
         "INSERT INTO my_table (col1, col2) VALUES ($1, $2)"

src/webserver/database/csv_import.rs

@@ -54,7 +54,7 @@ pub fn stream_query_results_with_conn<'a>(
                 ParsedStatement::CsvImport(csv_import) => {
                     let connection = take_connection(&request.app_state.db, db_connection).await?;
                     log::debug!("Executing CSV import: {csv_import:?}");
-                    run_csv_import(connection, csv_import, request).await.with_context(|| format!("Failed to import the CSV file {:?} into the table {:?}", csv_import.uploaded_file, csv_import.table_name))?;
+                    run_csv_import(connection, request.app_state.db.database_type, csv_import, request).await.with_context(|| format!("Failed to import the CSV file {:?} into the table {:?}", csv_import.uploaded_file, csv_import.table_name))?;
                 },
                 ParsedStatement::StmtWithParams(stmt) => {
                     let query = bind_parameters(stmt, request, db_connection).await?;