added progress bar and test script

Author: Thomas-Rauter

LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [Year] [Thomas Rauter]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -9,7 +9,6 @@ execute SQL queries on SQLite databases. The package supports organizing SQL que
 - [Usage](#usage)
   - [Basic Example](#basic-example)
   - [Directory Layouts](#directory-layouts)
-  - [Rerun Options](#rerun-options)
 - [Installation](#installation)
 
 ---

setup.py

@@ -13,6 +13,7 @@
     packages=find_packages(),
     install_requires=[
         "pandas>=2.0.0",
+        "tqdm>=4.5.0",
     ],
     python_requires=">=3.7",             # Minimum Python version
     classifiers=[                        # Metadata for PyPI

sqlite_query_manager/__pycache__/__init__.cpython-310.pyc

@@ -1,17 +1,19 @@
+from typing import Union, List
 import os
 import sqlite3
 import pandas as pd
 import logging
 from pathlib import Path
-from typing import Union, List
+from tqdm import tqdm
+import time
 
 
 def run_sql_queries(
-        query_dir: Union[str, os.PathLike],
-        db_file: Union[str, os.PathLike],
-        output_dir: Union[str, os.PathLike],
-        rerun_all: bool = False,
-        rerun_queries: List[str] = None
+    query_dir: Union[str, os.PathLike],
+    db_file: Union[str, os.PathLike],
+    output_dir: Union[str, os.PathLike],
+    rerun_all: bool = False,
+    rerun_queries: List[str] = None
 ) -> None:
     """
     Execute all SQL queries in a directory (including subdirectories) on a
@@ -44,78 +46,110 @@ def run_sql_queries(
         rerun_queries=rerun_queries
     )
 
-    # Normalize rerun_queries for comparison
     rerun_queries = set(rerun_queries or [])
-
-    # Configure logging
     logging.basicConfig(
         filename='query_manager.log',
         level=logging.INFO,
         format='%(asctime)s - %(levelname)s - %(message)s'
     )
     logger = logging.getLogger()
-
     logger.info("Starting SQL query execution process.")
 
-    # Connect to SQLite database
+    # Collect all SQL query files
+    query_files = []
+    for root, _, files in os.walk(query_dir):
+        for file_name in files:
+            if file_name.endswith('.sql'):
+                sql_file_path = os.path.join(
+                    root,
+                    file_name
+                )
+                query_files.append(sql_file_path)
+
+    # Filter queries based on rerun criteria
+    queries_to_execute = []
+    for sql_file_path in query_files:
+        relative_path = os.path.relpath(
+            os.path.dirname(sql_file_path),
+            query_dir
+        )
+
+        target_output_dir = os.path.join(
+            output_dir,
+            relative_path
+        )
+
+        os.makedirs(
+            target_output_dir,
+            exist_ok=True
+        )
+        output_file_path = os.path.join(
+            target_output_dir,
+            f"{Path(sql_file_path).stem}.csv"
+        )
+        if (
+            not os.path.exists(output_file_path) or
+            rerun_all or
+            Path(sql_file_path).name in rerun_queries
+        ):
+            queries_to_execute.append((
+                sql_file_path,
+                output_file_path
+            ))
+
+    if not queries_to_execute:
+        logger.info("No queries to execute. Exiting.")
+        print("No queries to execute.")
+        return
+
+    # Connect to the SQLite database
     try:
         conn = sqlite3.connect(db_file)
         logger.info(f"Connected to database: {db_file}")
     except Exception as e:
         logger.error(f"Failed to connect to database: {db_file}. Error: {e}")
         return
 
-    # Walk through the SQL directory structure
-    for root, _, files in os.walk(query_dir):
-        relative_path = os.path.relpath(root, query_dir)
-        target_output_dir = os.path.join(output_dir, relative_path)
+    # Progress bar setup
+    progress = tqdm(
+        total=len(queries_to_execute),
+        desc="Running queries",
+        unit=" query",
+        unit_scale=True
+    )
 
-        # Ensure output subdirectory exists
-        os.makedirs(target_output_dir, exist_ok=True)
+    execution_times = []
 
-        for file_name in files:
-            if file_name.endswith('.sql'):  # Process only .sql files
-                sql_file_path = os.path.join(root, file_name)
-                output_file_path = os.path.join(
-                    target_output_dir,
-                    f"{Path(file_name).stem}.csv"
-                )
+    for sql_file_path, output_file_path in queries_to_execute:
+        start_query_time = time.time()
+        try:
+            with open(sql_file_path, 'r') as query_file:
+                query = query_file.read()
+
+            logger.info(f"Executing query: {sql_file_path}")
+            df = pd.read_sql_query(query, conn)
+            df.to_csv(output_file_path, index=False)
+            logger.info(
+                f"Query executed successfully. Output saved to:"
+                f" {output_file_path}"
+            )
+
+        except Exception as e:
+            logger.error(f"Error executing query: {sql_file_path}. Error: {e}")
+
+        # Update progress and estimate remaining time
+        query_duration = time.time() - start_query_time
+        execution_times.append(query_duration)
+        avg_time_per_query = sum(execution_times) / len(execution_times)
+        remaining_queries = len(queries_to_execute) - progress.n - 1
+        estimated_time_left = avg_time_per_query * remaining_queries
+        progress.set_postfix_str(f"ETA: {estimated_time_left:.2f}s")
+        progress.update(1)
 
-                # Skip if output already exists and rerun_all is False,
-                # unless in rerun_queries
-                if (os.path.exists(output_file_path)
-                        and not rerun_all
-                        and file_name not in rerun_queries):
-                    logger.info(
-                        f"Skipping query (output exists):"
-                        f" {sql_file_path}"
-                    )
-                    continue
-
-                # Read and execute the SQL query
-                try:
-                    with open(sql_file_path, 'r') as query_file:
-                        query = query_file.read()
-
-                    logger.info(f"Executing query: {sql_file_path}")
-                    df = pd.read_sql_query(query, conn)
-
-                    # Save result to CSV
-                    df.to_csv(output_file_path, index=False)
-                    logger.info(
-                        f"Query executed successfully. Output saved to:"
-                        f" {output_file_path}"
-                    )
-
-                except Exception as e:
-                    logger.error(
-                        f"Error executing query:"
-                        f" {sql_file_path}. Error: {e}"
-                    )
-
-    # Close the database connection
+    progress.close()
     conn.close()
     logger.info("SQL query execution process completed.")
+    print("SQL query execution process completed.")
 
 
 def validate_inputs(
@@ -184,7 +218,7 @@ def validate_inputs(
 
     # No need to check if output_dir exists, as it may be created later
 
-    # Validate force_rerun
+    # Validate rerun_all
     if not isinstance(rerun_all, bool):
         raise TypeError(
             f"'force_rerun' must be a boolean."

sqlite_query_manager/__pycache__/run_sql_queries.cpython-310.pyc

@@ -0,0 +1,10 @@
+2025-01-01 13:36:35,140 - INFO - Starting SQL query execution process.
+2025-01-01 13:36:35,143 - INFO - Connected to database: test_sql\test_database.db
+2025-01-01 13:36:35,190 - INFO - Executing query: test_sql\queries\sample_query.sql
+2025-01-01 13:36:35,210 - INFO - Query executed successfully. Output saved to: test_sql\output\.\sample_query.csv
+2025-01-01 13:36:35,212 - INFO - SQL query execution process completed.
+2025-01-01 13:39:53,500 - INFO - Starting SQL query execution process.
+2025-01-01 13:39:53,502 - INFO - Connected to database: test_sql\test_database.db
+2025-01-01 13:39:53,532 - INFO - Executing query: test_sql\queries\sample_query.sql
+2025-01-01 13:39:53,546 - INFO - Query executed successfully. Output saved to: test_sql\output\.\sample_query.csv
+2025-01-01 13:39:53,547 - INFO - SQL query execution process completed.

sqlite_query_manager/run_sql_queries.py

@@ -0,0 +1,69 @@
+import sqlite3
+import os
+from sqlite_query_manager.run_sql_queries import run_sql_queries
+
+# Paths for testing
+TEST_DIR = "test_sql"
+DB_FILE = os.path.join(TEST_DIR, "test_database.db")
+QUERY_DIR = os.path.join(TEST_DIR, "queries")
+OUTPUT_DIR = os.path.join(TEST_DIR, "output")
+
+
+def create_test_environment():
+    """
+    Creates a minimal SQLite database, test query, and directory structure.
+    """
+    os.makedirs(QUERY_DIR, exist_ok=True)
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    # Create a sample SQLite database
+    conn = sqlite3.connect(DB_FILE)
+    cursor = conn.cursor()
+
+    # Create a sample table
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS test_table (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            age INTEGER
+        )
+    """)
+
+    # Insert some test data
+    cursor.executemany("""
+        INSERT INTO test_table (name, age) VALUES (?, ?)
+    """, [
+        ("Alice", 25),
+        ("Bob", 30),
+        ("Charlie", 35)
+    ])
+    conn.commit()
+    conn.close()
+
+    # Create a sample query file
+    with open(os.path.join(QUERY_DIR, "sample_query.sql"), "w") as f:
+        f.write("SELECT * FROM test_table;")
+
+
+def test_run_sql_queries():
+    """
+    Tests the run_sql_queries function with the created test environment.
+    """
+    run_sql_queries(
+        query_dir=QUERY_DIR,
+        db_file=DB_FILE,
+        output_dir=OUTPUT_DIR,
+        rerun_all=True
+    )
+
+    # Check the output
+    output_file = os.path.join(OUTPUT_DIR, "sample_query.csv")
+    if os.path.exists(output_file):
+        print("Test passed: Output file created.")
+    else:
+        print("Test failed: Output file not found.")
+
+
+if __name__ == "__main__":
+    create_test_environment()
+    test_run_sql_queries()

tests/query_manager.log

@@ -0,0 +1,7 @@
+id,name,age
+1,Alice,25
+2,Bob,30
+3,Charlie,35
+4,Alice,25
+5,Bob,30
+6,Charlie,35

tests/test_run_sql_queries.py

@@ -0,0 +1 @@
+SELECT * FROM test_table;