tests: User testcontainers for all Redis and PSQL tests, filter warnings

Author: fordN

pyproject.toml

@@ -53,7 +53,8 @@ delta_lake = [
 ]
 
 iceberg = [
-    "pyiceberg[sql-sqlite]>=0.9.1",
+    "pyiceberg[sql-sqlite]>=0.10.0",
+    "pydantic>=2.0,<2.12",  # PyIceberg 0.10.0 has issues with Pydantic 2.12+
 ]
 
 snowflake = [
@@ -68,7 +69,8 @@ all_loaders = [
     "psycopg2-binary>=2.9.0",       # PostgreSQL
     "redis>=4.5.0",                 # Redis
     "deltalake>=1.0.2",             # Delta Lake (consistent version)
-    "pyiceberg[sql-sqlite]>=0.9.1", # Apache Iceberg
+    "pyiceberg[sql-sqlite]>=0.10.0", # Apache Iceberg
+    "pydantic>=2.0,<2.12",          # PyIceberg 0.10.0 compatibility
     "snowflake-connector-python>=3.5.0",  # Snowflake
     "lmdb>=1.4.0",                  # LMDB
 ]
@@ -78,11 +80,11 @@ test = [
     "pytest-asyncio>=0.21.0",
     "pytest-mock>=3.10.0",
     "pytest-cov>=4.0.0",
-    "pytest-xdist>=3.0.0",         # Parallel test execution
-    "pytest-benchmark>=4.0.0",     # Performance benchmarking
-    "testcontainers>=3.7.0",       # Database containers for integration tests
-    "docker>=6.0.0",               # Required by testcontainers
-    "psutil>=5.9.0",               # Memory usage monitoring
+    "pytest-xdist>=3.0.0", # Parallel test execution
+    "pytest-benchmark>=4.0.0", # Performance benchmarking
+    "testcontainers>=4.0.0", # Database containers for integration tests
+    "docker>=6.0.0", # Required by testcontainers
+    "psutil>=5.9.0", # Memory usage monitoring
 ]
 
 [build-system]
@@ -98,6 +100,11 @@ addopts = [
     "--tb=short",
     "--strict-markers",
 ]
+filterwarnings = [
+    # Ignore testcontainers deprecation warnings from the library itself
+    "ignore:The @wait_container_is_ready decorator is deprecated:DeprecationWarning:testcontainers",
+    "ignore:The wait_for_logs function with string or callable predicates is deprecated:DeprecationWarning",
+]
 
 markers = [
     "unit: Unit tests (fast, no external dependencies)",

tests/conftest.py

@@ -23,6 +23,12 @@
 if 'TESTCONTAINERS_RYUK_DISABLED' not in os.environ:
     os.environ['TESTCONTAINERS_RYUK_DISABLED'] = 'true'
 
+# Set Docker host for Colima if not already set
+if 'DOCKER_HOST' not in os.environ:
+    colima_socket = Path.home() / '.colima' / 'default' / 'docker.sock'
+    if colima_socket.exists():
+        os.environ['DOCKER_HOST'] = f'unix://{colima_socket}'
+
 # Import testcontainers conditionally
 if USE_TESTCONTAINERS:
     try:
@@ -118,9 +124,19 @@ def postgres_container():
     if not TESTCONTAINERS_AVAILABLE:
         pytest.skip('Testcontainers not available')
 
+    import time
+
+    from testcontainers.core.waiting_utils import wait_for_logs
+
     container = PostgresContainer(image='postgres:13', username='test_user', password='test_pass', dbname='test_db')
     container.start()
 
+    # Wait for PostgreSQL to be ready using log message
+    wait_for_logs(container, 'database system is ready to accept connections', timeout=30)
+
+    # PostgreSQL logs "ready" twice - wait a bit more to ensure fully ready
+    time.sleep(2)
+
     yield container
 
     container.stop()
@@ -132,9 +148,14 @@ def redis_container():
     if not TESTCONTAINERS_AVAILABLE:
         pytest.skip('Testcontainers not available')
 
+    from testcontainers.core.waiting_utils import wait_for_logs
+
     container = RedisContainer(image='redis:7-alpine')
     container.start()
 
+    # Wait for Redis to be ready using log message
+    wait_for_logs(container, 'Ready to accept connections', timeout=30)
+
     yield container
 
     container.stop()

tests/performance/test_loader_performance.py

@@ -23,9 +23,9 @@
 class TestPostgreSQLPerformance:
     """Performance tests for PostgreSQL loader"""
 
-    def test_large_table_loading_performance(self, postgresql_config, performance_test_data, memory_monitor):
+    def test_large_table_loading_performance(self, postgresql_test_config, performance_test_data, memory_monitor):
         """Test loading large datasets with performance monitoring"""
-        loader = PostgreSQLLoader(postgresql_config)
+        loader = PostgreSQLLoader(postgresql_test_config)
 
         with loader:
             start_time = time.time()
@@ -59,7 +59,7 @@ def test_large_table_loading_performance(self, postgresql_config, performance_te
                 finally:
                     loader.pool.putconn(conn)
 
-    def test_batch_performance_scaling(self, postgresql_config, performance_test_data):
+    def test_batch_performance_scaling(self, postgresql_test_config, performance_test_data):
         """Test performance scaling with different batch processing approaches"""
         from src.amp.loaders.base import LoadMode
 
@@ -72,7 +72,7 @@ def test_batch_performance_scaling(self, postgresql_config, performance_test_dat
         results = {}
 
         for approach_name, batch_size in batch_approaches.items():
-            loader = PostgreSQLLoader(postgresql_config)
+            loader = PostgreSQLLoader(postgresql_test_config)
             table_name = f'perf_batch_{approach_name}'
 
             with loader:
@@ -123,9 +123,9 @@ def test_batch_performance_scaling(self, postgresql_config, performance_test_dat
         for approach, throughput in results.items():
             assert throughput > 500, f'{approach} too slow: {throughput:.0f} rows/sec'
 
-    def test_connection_pool_performance(self, postgresql_config, small_test_table):
+    def test_connection_pool_performance(self, postgresql_test_config, small_test_table):
         """Test connection pool efficiency under load"""
-        config = {**postgresql_config, 'max_connections': 5}
+        config = {**postgresql_test_config, 'max_connections': 5}
         loader = PostgreSQLLoader(config)
 
         with loader:
@@ -1016,7 +1016,7 @@ class TestCrossLoaderPerformance:
 
     def test_throughput_comparison(
         self,
-        postgresql_config,
+        postgresql_test_config,
         redis_test_config,
         snowflake_config,
         delta_basic_config,
@@ -1027,7 +1027,7 @@ def test_throughput_comparison(
         results = {}
 
         # Test PostgreSQL
-        pg_loader = PostgreSQLLoader(postgresql_config)
+        pg_loader = PostgreSQLLoader(postgresql_test_config)
         with pg_loader:
             start_time = time.time()
             result = pg_loader.load_table(medium_test_table, 'throughput_test')
@@ -1120,7 +1120,9 @@ def test_throughput_comparison(
             if throughput > 0:
                 print(f'  {loader_name}: {throughput:.0f}')
 
-    def test_memory_usage_comparison(self, postgresql_config, redis_test_config, snowflake_config, small_test_table):
+    def test_memory_usage_comparison(
+        self, postgresql_test_config, redis_test_config, snowflake_config, small_test_table
+    ):
         """Compare memory usage patterns across loaders"""
         try:
             import psutil
@@ -1132,7 +1134,7 @@ def test_memory_usage_comparison(self, postgresql_config, redis_test_config, sno
 
         # Test PostgreSQL memory usage
         initial_memory = process.memory_info().rss
-        pg_loader = PostgreSQLLoader(postgresql_config)
+        pg_loader = PostgreSQLLoader(postgresql_test_config)
         with pg_loader:
             pg_loader.load_table(small_test_table, 'memory_test')
             peak_memory = process.memory_info().rss