Merge branch 'master' into feature/webvtt-regression-test

Author: Rahul-2k4

database.py

@@ -4,15 +4,15 @@
 import re
 import traceback
 from abc import ABCMeta
-from typing import Any, Dict, Iterator, Tuple, Type, Union
+from typing import Any, Dict, Iterator, Optional, Tuple, Type, Union
 
 from sqlalchemy import create_engine
-from sqlalchemy.dialects.sqlite.pysqlite import SQLiteDialect_pysqlite
+from sqlalchemy.engine import Dialect
 from sqlalchemy.exc import SQLAlchemyError
-from sqlalchemy.ext.declarative import DeclarativeMeta, declarative_base
-from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.sql.schema import Column, Table
-from sqlalchemy.sql.sqltypes import Enum, SchemaType, TypeDecorator
+from sqlalchemy.orm import (DeclarativeBase, DeclarativeMeta, scoped_session,
+                            sessionmaker)
+from sqlalchemy.pool import StaticPool
+from sqlalchemy.sql.sqltypes import String, TypeDecorator
 
 from exceptions import EnumParsingException, FailedToSpawnDBSession
 
@@ -23,7 +23,12 @@ class DeclarativeABCMeta(DeclarativeMeta, ABCMeta):
     pass
 
 
-Base = declarative_base(metaclass=DeclarativeMeta)
+class Base(DeclarativeBase):
+    """Base class for all models."""
+
+    pass
+
+
 Base.query = None
 db_engine = None
 
@@ -43,9 +48,18 @@ def create_session(db_string: str, drop_tables: bool = False) -> scoped_session:
     global db_engine, Base
 
     try:
-        # In testing, we want to maintain same memory variable
-        if db_engine is None or 'TESTING' not in os.environ or os.environ['TESTING'] == 'False':
-            db_engine = create_engine(db_string, convert_unicode=True)
+        # Only create engine if it doesn't exist
+        # For SQLite in-memory, we must reuse the engine to share the database
+        if db_engine is None:
+            # For SQLite in-memory databases, use StaticPool to share connection
+            if db_string == 'sqlite:///:memory:':
+                db_engine = create_engine(
+                    db_string,
+                    connect_args={"check_same_thread": False},
+                    poolclass=StaticPool
+                )
+            else:
+                db_engine = create_engine(db_string)
         db_session = scoped_session(sessionmaker(bind=db_engine))
         Base.query = db_session.query_property()
 
@@ -162,32 +176,27 @@ def db_type(cls) -> DeclEnumType:
         return DeclEnumType(cls)
 
 
-class DeclEnumType(SchemaType, TypeDecorator):
+class DeclEnumType(TypeDecorator):
     """Declarative enumeration type."""
 
     cache_ok = True
+    impl = String(50)
 
     def __init__(self, enum: Any) -> None:
         self.enum = enum
-        self.impl = Enum(
-            *enum.values(),
-            name="ck{0}".format(re.sub('([A-Z])', lambda m: "_" + m.group(1).lower(), enum.__name__))
-        )
-
-    def _set_table(self, table: Column, column: Table) -> None:
-        self.impl._set_table(table, column)
+        super().__init__()
 
-    def copy(self) -> DeclEnumType:
+    def copy(self, **kwargs: Any) -> DeclEnumType:
         """Get enumeration type of self."""
         return DeclEnumType(self.enum)
 
-    def process_bind_param(self, value: EnumSymbol, dialect: SQLiteDialect_pysqlite) -> str:
+    def process_bind_param(self, value: Optional[EnumSymbol], dialect: Dialect) -> Optional[str]:
         """Get process bind parameter."""
         if value is None:
             return None
         return value.value
 
-    def process_result_value(self, value: str, dialect: SQLiteDialect_pysqlite) -> EnumSymbol:
+    def process_result_value(self, value: Optional[str], dialect: Dialect) -> Optional[EnumSymbol]:
         """Get process result value."""
         if value is None:
             return None

install/deploy/pre_deploy.sh

@@ -101,6 +101,32 @@ if ! git diff --quiet 2>/dev/null; then
     git status --short
 fi
 
+# Check 8: Verify logs directory ownership
+LOGS_DIR="$INSTALL_FOLDER/logs"
+WEB_USER="${WEB_USER:-www-data}"
+if [ -d "$LOGS_DIR" ]; then
+    LOGS_OWNER=$(stat -c '%U' "$LOGS_DIR" 2>/dev/null || echo "unknown")
+    if [ "$LOGS_OWNER" != "$WEB_USER" ]; then
+        echo "WARNING: Logs directory owned by '$LOGS_OWNER', should be '$WEB_USER'"
+        echo "Fixing ownership..."
+        chown -R "$WEB_USER:$WEB_USER" "$LOGS_DIR" 2>/dev/null || {
+            echo "ERROR: Failed to fix logs ownership. Run manually:"
+            echo "  sudo chown -R $WEB_USER:$WEB_USER $LOGS_DIR"
+            exit 1
+        }
+        echo "✓ Logs directory ownership fixed"
+    else
+        echo "✓ Logs directory ownership OK ($WEB_USER)"
+    fi
+else
+    echo "Creating logs directory with correct ownership..."
+    mkdir -p "$LOGS_DIR"
+    chown "$WEB_USER:$WEB_USER" "$LOGS_DIR" 2>/dev/null || {
+        echo "WARNING: Could not set logs ownership (run as root)"
+    }
+    echo "✓ Logs directory created"
+fi
+
 # Export backup directory for other scripts
 echo "$BACKUP_DIR" > /tmp/sp-deploy-backup-dir.txt
 

log_configuration.py

@@ -3,9 +3,10 @@
 import logging
 import logging.handlers
 import os
+import sys
 from logging import Logger, StreamHandler
 from logging.handlers import RotatingFileHandler
-from typing import Union
+from typing import Optional, Union
 
 
 class LogConfiguration:
@@ -19,21 +20,47 @@ def __init__(self, folder: str, filename: str, debug: bool = False) -> None:
             self._consoleLogger.setLevel(logging.DEBUG)
         else:
             self._consoleLogger.setLevel(logging.INFO)
-        # create a file handler
-        path = os.path.join(folder, 'logs', f'{filename}.log')
-        self._fileLogger = logging.handlers.RotatingFileHandler(path, maxBytes=1024 * 1024, backupCount=20)
-        self._fileLogger.setLevel(logging.DEBUG)
-        # create a logging format
-        formatter = logging.Formatter('[%(name)s][%(levelname)s][%(asctime)s] %(message)s')
-        self._fileLogger.setFormatter(formatter)
+
+        # create a file handler with permission error handling
+        self._fileLogger: Optional[RotatingFileHandler] = None
+        log_dir = os.path.join(folder, 'logs')
+        path = os.path.join(log_dir, f'{filename}.log')
+
+        try:
+            # Ensure logs directory exists
+            os.makedirs(log_dir, exist_ok=True)
+
+            self._fileLogger = logging.handlers.RotatingFileHandler(
+                path, maxBytes=1024 * 1024, backupCount=20
+            )
+            self._fileLogger.setLevel(logging.DEBUG)
+            # create a logging format
+            formatter = logging.Formatter('[%(name)s][%(levelname)s][%(asctime)s] %(message)s')
+            self._fileLogger.setFormatter(formatter)
+        except PermissionError as e:
+            # Log file owned by different user (e.g., root vs www-data)
+            # Fall back to console-only logging rather than crashing
+            print(
+                f"[WARNING] Cannot write to log file {path}: {e}. "
+                f"Falling back to console-only logging. "
+                f"Fix: sudo chown www-data:www-data {log_dir} -R",
+                file=sys.stderr
+            )
+        except OSError as e:
+            # Other filesystem errors (disk full, etc.)
+            print(
+                f"[WARNING] Cannot create log file {path}: {e}. "
+                f"Falling back to console-only logging.",
+                file=sys.stderr
+            )
 
     @property
-    def file_logger(self) -> RotatingFileHandler:
+    def file_logger(self) -> Optional[RotatingFileHandler]:
         """
         Get file logger.
 
-        :return: file logger
-        :rtype: logging.handlers.RotatingFileHandler
+        :return: file logger or None if file logging unavailable
+        :rtype: Optional[logging.handlers.RotatingFileHandler]
         """
         return self._fileLogger
 
@@ -59,7 +86,8 @@ def create_logger(self, name: str) -> Logger:
         logger = logging.getLogger(name)
         logger.setLevel(logging.DEBUG)
         # add the handlers to the logger
-        logger.addHandler(self.file_logger)
-        logger.addHandler(self.console_logger)
+        if self._fileLogger is not None:
+            logger.addHandler(self._fileLogger)
+        logger.addHandler(self._consoleLogger)
 
         return logger

mod_health/controllers.py

@@ -6,6 +6,7 @@
 from typing import Any, Dict, Optional, Tuple
 
 from flask import Blueprint, current_app, jsonify
+from sqlalchemy import text
 
 mod_health = Blueprint('health', __name__)
 
@@ -20,7 +21,7 @@ def check_database() -> Dict[str, Any]:
     try:
         from database import create_session
         db = create_session(current_app.config['DATABASE_URI'])
-        db.execute('SELECT 1')
+        db.execute(text('SELECT 1'))
         # remove() returns the scoped session's connection to the pool
         db.remove()
         return {'status': 'ok'}

mod_test/controllers.py

@@ -1,19 +1,16 @@
 """Logic to find all tests, their progress and details of individual test."""
 
 import os
-from datetime import datetime
 from typing import Any, Dict, List
 
 from flask import (Blueprint, Response, abort, g, jsonify, redirect, request,
                    url_for)
-from sqlalchemy import and_, func
-from sqlalchemy.sql import label
+from sqlalchemy import and_
 
 from decorators import template_renderer
 from exceptions import TestNotFoundException
 from mod_auth.controllers import check_access_rights, login_required
 from mod_auth.models import Role
-from mod_ci.models import GcpInstance
 from mod_customized.models import TestFork
 from mod_home.models import CCExtractorVersion, GeneralData
 from mod_regression.models import (Category, RegressionTestOutput,
@@ -133,58 +130,44 @@ def get_data_for_test(test, title=None) -> Dict[str, Any]:
     if title is None:
         title = f"test {test.id}"
 
-    hours = 0.00
-    minutes = 0.00
-    queued_tests = 0
-
-    """
-    evaluating estimated time if the test is still in queue
-    estimated time = (number of tests already in queue + 1) * (average time of that platform)
-                      - (time already spend by those tests)
-    calculates time in minutes and hours
-    """
+    # Calculate average runtime for this platform (used when test hasn't started yet)
+    avg_minutes = 0
     if len(test.progress) == 0:
-        var_average = 'average_time_' + test.platform.value
-
-        # get average build and prep time.
-        prep_average_key = 'avg_prep_time_' + test.platform.value
-        average_prep_time = int(float(GeneralData.query.filter(GeneralData.key == prep_average_key).first().value))
-
-        test_progress_last_entry = g.db.query(func.max(TestProgress.test_id)).first()
-        last_test_id = test_progress_last_entry[0] if test_progress_last_entry is not None else 0
-        queued_gcp_instance = g.db.query(GcpInstance.test_id).filter(GcpInstance.test_id < test.id).subquery()
-        queued_gcp_instance_entries = g.db.query(Test.id).filter(
-            and_(Test.id.in_(queued_gcp_instance), Test.platform == test.platform)
-        ).subquery()
-        gcp_instance_test = g.db.query(TestProgress.test_id, label('time', func.group_concat(
-            TestProgress.timestamp))).filter(TestProgress.test_id.in_(queued_gcp_instance_entries)).group_by(
-            TestProgress.test_id).all()
-        number_gcp_instance_test = g.db.query(Test.id).filter(
-            and_(Test.id > last_test_id, Test.id < test.id, Test.platform == test.platform)
-        ).count()
-        average_duration = float(GeneralData.query.filter(GeneralData.key == var_average).first().value)
-        queued_tests = number_gcp_instance_test
-        time_run = 0.00
-        for pr_test in gcp_instance_test:
-            timestamps = pr_test.time.split(',')
-            start = datetime.strptime(timestamps[0], '%Y-%m-%d %H:%M:%S')
-            end = datetime.strptime(timestamps[-1], '%Y-%m-%d %H:%M:%S')
-            time_run += (end - start).total_seconds()
-        # subtracting current running tests
-        total = average_prep_time + average_duration - time_run
-        minutes = (total % 3600) // 60
-        hours = total // 3600
+        try:
+            avg_time_key = 'average_time_' + test.platform.value
+            prep_time_key = 'avg_prep_time_' + test.platform.value
+
+            avg_time_record = GeneralData.query.filter(GeneralData.key == avg_time_key).first()
+            prep_time_record = GeneralData.query.filter(GeneralData.key == prep_time_key).first()
+
+            avg_duration = float(avg_time_record.value) if avg_time_record else 0
+            avg_prep = float(prep_time_record.value) if prep_time_record else 0
+
+            # Total average time in minutes
+            avg_minutes = int((avg_duration + avg_prep) / 60)
+        except (ValueError, AttributeError):
+            avg_minutes = 0
 
     results = get_test_results(test)
 
+    # Calculate sample progress for initial page load
+    completed_samples = len(test.results)
+    total_samples = len(test.get_customized_regressiontests())
+    progress_percentage = 0
+    if total_samples > 0:
+        progress_percentage = int((completed_samples / total_samples) * 100)
+
     return {
         'test': test,
         'TestType': TestType,
         'results': results,
         'title': title,
-        'next': queued_tests,
-        'min': minutes,
-        'hr': hours
+        'avg_minutes': avg_minutes,
+        'sample_progress': {
+            'current': completed_samples,
+            'total': total_samples,
+            'percentage': progress_percentage
+        }
     }
 
 
@@ -212,11 +195,23 @@ def get_json_data(test_id):
             'message': entry.message
         })
 
+    # Calculate sample progress from existing TestResult data
+    completed_samples = len(test.results)
+    total_samples = len(test.get_customized_regressiontests())
+    progress_percentage = 0
+    if total_samples > 0:
+        progress_percentage = int((completed_samples / total_samples) * 100)
+
     return jsonify({
         'status': 'success',
         'details': pr_data["progress"],
         'complete': test.finished,
-        'progress_array': progress_array
+        'progress_array': progress_array,
+        'sample_progress': {
+            'current': completed_samples,
+            'total': total_samples,
+            'percentage': progress_percentage
+        }
     })
 
 

mypy.ini

@@ -1,5 +1,14 @@
 [mypy]
-python_version = 3.8
+python_version = 3.10
 ignore_missing_imports = True
-warn_unused_ignores = True
+warn_unused_ignores = False
 exclude = venv*
+
+# Disable errors for SQLAlchemy 2.0 migration
+# These require more extensive refactoring:
+# - attr-defined: Model.query is set dynamically at runtime
+# - var-annotated: DeclEnum columns need proper type annotations
+# - assignment: datetime/date default values with Column types
+# - arg-type: Column types vs primitive types in function calls
+# - index: Test model indexing issues
+disable_error_code = attr-defined, var-annotated, assignment, arg-type, index