Merge pull request #25 from patchmemory/task/combined-folder-config-and-logs-pr

feat: folder-config precedence + logs endpoint + ipynb streaming

Author: patchmemory

scidk/app.py

@@ -5001,6 +5001,80 @@ def api_metrics():
         except Exception as e:
             return jsonify({'error': str(e)}), 500
 
+    @api.get('/logs')
+    def api_logs():
+        """
+        Browse operational logs with pagination and filters.
+        Query params: limit, offset, level, since_ts
+        Privacy: No sensitive file paths or user data exposed.
+        """
+        try:
+            from .core import path_index_sqlite as pix
+            limit = min(int(request.args.get('limit', 100)), 1000)
+            offset = int(request.args.get('offset', 0))
+            level = request.args.get('level', '').strip().upper() or None
+            since_ts = request.args.get('since_ts', '').strip() or None
+
+            conn = pix.connect()
+            try:
+                cur = conn.cursor()
+                # Build query with filters
+                conditions = []
+                params = []
+                if level:
+                    conditions.append("level = ?")
+                    params.append(level)
+                if since_ts:
+                    try:
+                        ts_val = float(since_ts)
+                        conditions.append("ts >= ?")
+                        params.append(ts_val)
+                    except Exception:
+                        pass
+
+                where_clause = ""
+                if conditions:
+                    where_clause = " WHERE " + " AND ".join(conditions)
+
+                # Get total count
+                count_query = f"SELECT COUNT(*) FROM logs{where_clause}"
+                cur.execute(count_query, params)
+                row = cur.fetchone()
+                total = row[0] if row else 0
+
+                # Get logs (most recent first)
+                query = f"""
+                    SELECT ts, level, message, context
+                    FROM logs{where_clause}
+                    ORDER BY ts DESC
+                    LIMIT ? OFFSET ?
+                """
+                cur.execute(query, params + [limit, offset])
+                rows = cur.fetchall()
+
+                logs = []
+                for row in rows:
+                    logs.append({
+                        'ts': row[0],
+                        'level': row[1],
+                        'message': row[2],
+                        'context': row[3]
+                    })
+
+                return jsonify({
+                    'logs': logs,
+                    'total': total,
+                    'limit': limit,
+                    'offset': offset
+                }), 200
+            finally:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
+        except Exception as e:
+            return jsonify({'error': str(e)}), 500
+
     # Rclone interpretation settings (GET/POST)
     @api.get('/settings/rclone-interpret')
     def api_settings_rclone_interpret_get():

scidk/core/folder_config.py

@@ -2,7 +2,11 @@
 from pathlib import Path
 from typing import Dict, Any, Optional
 
-import tomllib  # Python 3.11+ (built-in)
+# Python 3.11+ has tomllib built-in, 3.10 needs tomli backport
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore
 
 DEFAULTS = {
     'include': [],  # list of glob patterns

scidk/services/scans_service.py

@@ -226,20 +226,9 @@ def _dir_signature(dir_path: Path):
                         key = str(dpath.resolve())
                         conf = _conf_cache.get(key)
                         if conf is None:
-                            # Prefer local .scidk.toml in this directory (closest wins), then fall back to effective config
+                            # Use load_effective_config to properly honor per-folder precedence
                             try:
-                                tpath = Path(dpath) / '.scidk.toml'
-                                if tpath.exists():
-                                    import tomllib as _toml
-                                    try:
-                                        data = _toml.loads(tpath.read_text(encoding='utf-8'))
-                                    except Exception:
-                                        data = {}
-                                    inc = data.get('include') if isinstance(data.get('include'), list) else []
-                                    exc = data.get('exclude') if isinstance(data.get('exclude'), list) else []
-                                    conf = {'include': [str(x) for x in inc], 'exclude': [str(x) for x in exc], 'interpreters': None}
-                                else:
-                                    conf = load_effective_config(dpath, stop_at=base)
+                                conf = load_effective_config(dpath, stop_at=base)
                             except Exception:
                                 conf = {'include': [], 'exclude': [], 'interpreters': None}
                             _conf_cache[key] = conf

tests/test_logs_endpoint.py

@@ -0,0 +1,119 @@
+import time
+from scidk.app import create_app
+
+
+def test_logs_endpoint_exists():
+    """Test that /api/logs endpoint exists and returns expected structure."""
+    app = create_app()
+    app.config['TESTING'] = True
+    with app.test_client() as c:
+        r = c.get('/api/logs')
+        assert r.status_code == 200
+        data = r.get_json()
+        # Expect keys present
+        assert 'logs' in data
+        assert 'total' in data
+        assert 'limit' in data
+        assert 'offset' in data
+        assert isinstance(data['logs'], list)
+
+
+def test_logs_endpoint_pagination():
+    """Test that pagination parameters work correctly."""
+    app = create_app()
+    app.config['TESTING'] = True
+    with app.test_client() as c:
+        # Test with custom limit and offset
+        r = c.get('/api/logs?limit=5&offset=0')
+        assert r.status_code == 200
+        data = r.get_json()
+        assert data['limit'] == 5
+        assert data['offset'] == 0
+        assert len(data['logs']) <= 5
+
+
+def test_logs_endpoint_level_filter():
+    """Test that level filter works correctly."""
+    app = create_app()
+    app.config['TESTING'] = True
+    with app.test_client() as c:
+        # Insert a test log entry
+        from scidk.core import path_index_sqlite as pix
+        conn = pix.connect()
+        try:
+            from scidk.core import migrations as _migs
+            _migs.migrate(conn)
+            cur = conn.cursor()
+            cur.execute(
+                "INSERT INTO logs (ts, level, message, context) VALUES (?, ?, ?, ?)",
+                (time.time(), 'ERROR', 'Test error message', None)
+            )
+            conn.commit()
+        finally:
+            try:
+                conn.close()
+            except Exception:
+                pass
+
+        # Query with level filter
+        r = c.get('/api/logs?level=ERROR')
+        assert r.status_code == 200
+        data = r.get_json()
+        # Should have at least our test error
+        error_logs = [log for log in data['logs'] if log['level'] == 'ERROR']
+        assert len(error_logs) > 0
+
+
+def test_logs_endpoint_since_ts_filter():
+    """Test that since_ts filter works correctly."""
+    app = create_app()
+    app.config['TESTING'] = True
+    with app.test_client() as c:
+        # Insert test log entries with different timestamps
+        from scidk.core import path_index_sqlite as pix
+        conn = pix.connect()
+        try:
+            from scidk.core import migrations as _migs
+            _migs.migrate(conn)
+            cur = conn.cursor()
+            now = time.time()
+            cur.execute(
+                "INSERT INTO logs (ts, level, message, context) VALUES (?, ?, ?, ?)",
+                (now - 100, 'INFO', 'Old log', None)
+            )
+            cur.execute(
+                "INSERT INTO logs (ts, level, message, context) VALUES (?, ?, ?, ?)",
+                (now, 'INFO', 'Recent log', None)
+            )
+            conn.commit()
+        finally:
+            try:
+                conn.close()
+            except Exception:
+                pass
+
+        # Query with since_ts filter (only recent logs)
+        cutoff = time.time() - 50
+        r = c.get(f'/api/logs?since_ts={cutoff}')
+        assert r.status_code == 200
+        data = r.get_json()
+        # All returned logs should be after cutoff
+        for log in data['logs']:
+            assert log['ts'] >= cutoff
+
+
+def test_logs_endpoint_no_sensitive_data():
+    """Test that logs don't expose sensitive file paths or user data."""
+    app = create_app()
+    app.config['TESTING'] = True
+    with app.test_client() as c:
+        r = c.get('/api/logs')
+        assert r.status_code == 200
+        data = r.get_json()
+        # Verify response structure contains only safe fields
+        for log in data['logs']:
+            assert set(log.keys()) == {'ts', 'level', 'message', 'context'}
+            # No 'user', 'password', 'secret', etc. fields
+            assert 'user' not in log
+            assert 'password' not in log
+            assert 'secret' not in log