From 88e80302d7c30f82c5be4a1ae3c0d725a908b6c1 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 12 Feb 2026 11:31:13 +0100 Subject: [PATCH 01/28] feat: improve logs view with keyset pagination, column selection, and sticky chart - Replace OFFSET-based pagination with cursor-based keyset pagination using timestamp for consistent, performant paging (#84) - Select only visible columns instead of SELECT * for faster log loading; fetch full row on demand for detail modal (#87) - Pin chart to viewport top when scrolling logs with collapse toggle and bidirectional scroll/scrubber synchronization (#66) Closes #84, closes #87, closes #66 Co-Authored-By: Claude Opus 4.6 --- css/chart.css | 44 +++++++ css/modals.css | 7 ++ dashboard.html | 1 + js/chart.js | 36 ++++++ js/columns.js | 24 ++++ js/columns.test.js | 71 +++++++++++ js/dashboard-init.js | 13 +- js/logs.js | 243 ++++++++++++++++++++++++++++++++----- js/pagination.js | 11 +- js/pagination.test.js | 75 +++++++++--- js/sql-loader.js | 1 + sql/queries/log-detail.sql | 4 + sql/queries/logs-more.sql | 5 +- sql/queries/logs.sql | 2 +- 14 files changed, 481 insertions(+), 56 deletions(-) create mode 100644 js/columns.test.js create mode 100644 sql/queries/log-detail.sql diff --git a/css/chart.css b/css/chart.css index 1098c15..d73319c 100644 --- a/css/chart.css +++ b/css/chart.css @@ -9,6 +9,50 @@ margin: 0 -24px 24px -24px; } +/* Sticky chart when logs view is active */ +.logs-active .chart-section { + position: sticky; + top: 0; + z-index: 20; +} + +/* Collapse/expand toggle */ +.chart-collapse-toggle { + display: none; + width: 100%; + height: 24px; + border: none; + background: var(--chart-bg); + border-top: 1px solid var(--border); + cursor: pointer; + padding: 0; + color: var(--text-secondary); + font-size: 10px; + line-height: 24px; + text-align: center; + opacity: 0.6; + transition: opacity 0.15s ease; +} + +.chart-collapse-toggle:hover { + opacity: 1; +} + +.logs-active .chart-collapse-toggle { + display: block; +} + +/* Collapsed state */ +.chart-section.chart-collapsed .chart-container { + height: 0; + overflow: hidden; + transition: height 0.2s ease; +} + +.chart-section:not(.chart-collapsed) .chart-container { + transition: height 0.2s ease; +} + @media (max-width: 600px) { .chart-section { margin: 0 -12px 16px -12px; diff --git a/css/modals.css b/css/modals.css index 981fc7e..bbfff4b 100644 --- a/css/modals.css +++ b/css/modals.css @@ -365,6 +365,13 @@ margin-bottom: 12px; } +.log-detail-loading { + padding: 40px 20px; + text-align: center; + color: var(--text-secondary); + font-size: 14px; +} + @media (max-width: 600px) { #logDetailModal { width: 100vw; diff --git a/dashboard.html b/dashboard.html index b51ccb8..b02aced 100644 --- a/dashboard.html +++ b/dashboard.html @@ -80,6 +80,7 @@

Requests over time

+ diff --git a/js/chart.js b/js/chart.js index a2a940f..2736178 100644 --- a/js/chart.js +++ b/js/chart.js @@ -90,6 +90,34 @@ let isDragging = false; let dragStartX = null; let justCompletedDrag = false; +// Callback for chart→scroll sync (set by logs.js) +let onChartHoverTimestamp = null; + +/** + * Set callback for chart hover → scroll sync + * @param {Function} callback - Called with timestamp when hovering chart in logs view + */ +export function setOnChartHoverTimestamp(callback) { + onChartHoverTimestamp = callback; +} + +/** + * Position the scrubber line at a given timestamp (called from logs.js scroll sync) + * @param {Date} timestamp - Timestamp to position scrubber at + */ +export function setScrubberPosition(timestamp) { + if (!scrubberLine) return; + const x = getXAtTime(timestamp); + const chartLayout = getChartLayout(); + if (!chartLayout) return; + + const { padding, height } = chartLayout; + scrubberLine.style.left = `${x}px`; + scrubberLine.style.top = `${padding.top}px`; + scrubberLine.style.height = `${height - padding.top - padding.bottom}px`; + scrubberLine.classList.add('visible'); +} + /** * Initialize canvas for chart rendering */ @@ -740,6 +768,14 @@ export function setupChartNavigation(callback) { const y = e.clientY - rect.top; updateScrubber(x, y); + // Chart→Scroll sync: when hovering chart in logs view, scroll to matching time + if (onChartHoverTimestamp && state.showLogs) { + const hoverTime = getTimeAtX(x); + if (hoverTime) { + onChartHoverTimestamp(hoverTime); + } + } + // Ship tooltip on hover (handled here since nav overlay captures canvas events) const ship = getShipAtPoint(getShipPositions(), x, y); if (ship) { diff --git a/js/columns.js b/js/columns.js index c8ed1a3..524d7c1 100644 --- a/js/columns.js +++ b/js/columns.js @@ -181,3 +181,27 @@ export const LOG_COLUMN_SHORT_LABELS = Object.fromEntries( .filter((def) => def.shortLabel) .map((def) => [def.logKey, def.shortLabel]), ); + +/** + * Columns always included in logs queries (needed for internal use, not display). + * @type {string[]} + */ +const ALWAYS_NEEDED_COLUMNS = ['timestamp', 'source', 'sample_hash']; + +/** + * Build the backtick-quoted column list for logs SQL queries. + * Includes LOG_COLUMN_ORDER columns plus always-needed columns plus any pinned columns. + * @param {string[]} [pinnedColumns] - Additional pinned columns to include. + * @returns {string} Comma-separated, backtick-quoted column list for SQL SELECT. + */ +export function buildLogColumnsSql(pinnedColumns = []) { + const seen = new Set(); + const cols = []; + for (const col of [...ALWAYS_NEEDED_COLUMNS, ...LOG_COLUMN_ORDER, ...pinnedColumns]) { + if (!seen.has(col)) { + seen.add(col); + cols.push(`\`${col}\``); + } + } + return cols.join(', '); +} diff --git a/js/columns.test.js b/js/columns.test.js new file mode 100644 index 0000000..6b9655c --- /dev/null +++ b/js/columns.test.js @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { assert } from 'chai'; +import { buildLogColumnsSql, LOG_COLUMN_ORDER } from './columns.js'; + +describe('buildLogColumnsSql', () => { + it('returns backtick-quoted column list', () => { + const result = buildLogColumnsSql(); + assert.include(result, '`timestamp`'); + assert.include(result, '`request.host`'); + assert.include(result, '`response.status`'); + }); + + it('includes always-needed columns', () => { + const result = buildLogColumnsSql(); + assert.include(result, '`timestamp`'); + assert.include(result, '`source`'); + assert.include(result, '`sample_hash`'); + }); + + it('includes all LOG_COLUMN_ORDER columns', () => { + const result = buildLogColumnsSql(); + for (const col of LOG_COLUMN_ORDER) { + assert.include(result, `\`${col}\``, `missing column: ${col}`); + } + }); + + it('does not duplicate columns', () => { + // timestamp is in both ALWAYS_NEEDED and LOG_COLUMN_ORDER + const result = buildLogColumnsSql(); + const matches = result.match(/`timestamp`/g); + assert.strictEqual(matches.length, 1, 'timestamp should appear exactly once'); + }); + + it('includes pinned columns', () => { + const result = buildLogColumnsSql(['custom.column']); + assert.include(result, '`custom.column`'); + }); + + it('does not duplicate pinned columns already in the list', () => { + const result = buildLogColumnsSql(['request.host']); + const matches = result.match(/`request\.host`/g); + assert.strictEqual(matches.length, 1, 'request.host should appear exactly once'); + }); + + it('returns comma-separated values', () => { + const result = buildLogColumnsSql(); + const parts = result.split(', '); + assert.ok(parts.length > 5, 'should have many columns'); + for (const part of parts) { + assert.match(part, /^`[^`]+`$/, `each part should be backtick-quoted: ${part}`); + } + }); + + it('starts with always-needed columns', () => { + const result = buildLogColumnsSql(); + const parts = result.split(', '); + assert.strictEqual(parts[0], '`timestamp`'); + assert.strictEqual(parts[1], '`source`'); + assert.strictEqual(parts[2], '`sample_hash`'); + }); +}); diff --git a/js/dashboard-init.js b/js/dashboard-init.js index e561242..febdd18 100644 --- a/js/dashboard-init.js +++ b/js/dashboard-init.js @@ -29,7 +29,7 @@ import { } from './timer.js'; import { loadTimeSeries, setupChartNavigation, getDetectedAnomalies, getLastChartData, - renderChart, + renderChart, setOnChartHoverTimestamp, } from './chart.js'; import { loadAllBreakdowns, loadBreakdown, getBreakdowns, markSlowestFacet, resetFacetTimings, @@ -40,7 +40,7 @@ import { getFilterForValue, } from './filters.js'; import { - loadLogs, toggleLogsView, setLogsElements, setOnShowFiltersView, + loadLogs, toggleLogsView, setLogsElements, setOnShowFiltersView, scrollLogsToTimestamp, } from './logs.js'; import { loadHostAutocomplete } from './autocomplete.js'; import { initModal, closeQuickLinksModal } from './modal.js'; @@ -202,6 +202,15 @@ export function initDashboard(config = {}) { } }); + // Chart→Scroll sync: throttled to avoid excessive scrolling + let lastHoverScroll = 0; + setOnChartHoverTimestamp((timestamp) => { + const now = Date.now(); + if (now - lastHoverScroll < 300) return; + lastHoverScroll = now; + scrollLogsToTimestamp(timestamp); + }); + setFilterCallbacks(saveStateToURL, loadDashboard); setOnBeforeRestore(() => invalidateInvestigationCache()); setOnStateRestored(loadDashboard); diff --git a/js/logs.js b/js/logs.js index 9a13d2b..6b75bd2 100644 --- a/js/logs.js +++ b/js/logs.js @@ -18,10 +18,12 @@ import { escapeHtml } from './utils.js'; import { formatBytes } from './format.js'; import { getColorForColumn } from './colors/index.js'; import { getRequestContext, isRequestCurrent } from './request-context.js'; -import { LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS } from './columns.js'; +import { LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, buildLogColumnsSql } from './columns.js'; import { loadSql } from './sql-loader.js'; import { buildLogRowHtml, buildLogTableHeaderHtml } from './templates/logs-table.js'; import { PAGE_SIZE, PaginationState } from './pagination.js'; +import { setScrubberPosition } from './chart.js'; +import { parseUTC } from './chart-state.js'; /** * Build ordered log column list from available columns. @@ -241,41 +243,101 @@ export function closeLogDetailModal() { } } +/** + * Show loading state in the detail modal. + */ +function showDetailLoading() { + const table = document.getElementById('logDetailTable'); + if (table) { + table.innerHTML = 'Loading full row data\u2026'; + } +} + +/** + * Fetch full row data for a single log entry. + * @param {Object} partialRow - Row with at least timestamp and request.host + * @returns {Promise} Full row data or null on failure + */ +async function fetchFullRow(partialRow) { + const { timestamp } = partialRow; + const host = partialRow['request.host'] || ''; + const sql = await loadSql('log-detail', { + database: DATABASE, + table: getTable(), + timestamp: String(timestamp), + host: host.replace(/'/g, "\\'"), + }); + const result = await query(sql); + return result.data.length > 0 ? result.data[0] : null; +} + +/** + * Initialize the log detail modal element and event listeners. + */ +function initLogDetailModal() { + if (logDetailModal) return; + logDetailModal = document.getElementById('logDetailModal'); + if (!logDetailModal) return; + + // Close on backdrop click + logDetailModal.addEventListener('click', (e) => { + if (e.target === logDetailModal) { + closeLogDetailModal(); + } + }); + + // Close on Escape + logDetailModal.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { + closeLogDetailModal(); + } + }); + + // Close button handler + const closeBtn = logDetailModal.querySelector('[data-action="close-log-detail"]'); + if (closeBtn) { + closeBtn.addEventListener('click', closeLogDetailModal); + } +} + /** * Open log detail modal for a row. + * Fetches full row data on demand if not already present. * @param {number} rowIdx */ -export function openLogDetailModal(rowIdx) { +export async function openLogDetailModal(rowIdx) { const row = state.logsData[rowIdx]; if (!row) return; - if (!logDetailModal) { - logDetailModal = document.getElementById('logDetailModal'); - if (!logDetailModal) return; + initLogDetailModal(); + if (!logDetailModal) return; - // Close on backdrop click - logDetailModal.addEventListener('click', (e) => { - if (e.target === logDetailModal) { - closeLogDetailModal(); - } - }); + // Show modal immediately with loading state + showDetailLoading(); + logDetailModal.showModal(); - // Close on Escape - logDetailModal.addEventListener('keydown', (e) => { - if (e.key === 'Escape') { - closeLogDetailModal(); - } - }); + // Check if row already has full data (e.g. from a previous fetch) + if (row.fullRowData) { + renderLogDetailContent(row.fullRowData); + return; + } - // Close button handler - const closeBtn = logDetailModal.querySelector('[data-action="close-log-detail"]'); - if (closeBtn) { - closeBtn.addEventListener('click', closeLogDetailModal); + try { + const fullRow = await fetchFullRow(row); + if (fullRow) { + // Cache the full row for future opens + row.fullRowData = fullRow; + renderLogDetailContent(fullRow); + } else { + // Fallback: render with partial data + renderLogDetailContent(row); } + } catch (err) { + // eslint-disable-next-line no-console + console.error('Failed to fetch full row:', err); + // Fallback: render with partial data + renderLogDetailContent(row); } - - renderLogDetailContent(row); - logDetailModal.showModal(); } // Copy row data as JSON when clicking on row background @@ -426,12 +488,13 @@ async function loadMoreLogs() { const sql = await loadSql('logs-more', { database: DATABASE, table: getTable(), + columns: buildLogColumnsSql(state.pinnedColumns), timeFilter, hostFilter, facetFilters, additionalWhereClause: state.additionalWhereClause, pageSize: String(PAGE_SIZE), - offset: String(pagination.offset), + cursor: pagination.cursor, }); try { @@ -441,7 +504,7 @@ async function loadMoreLogs() { state.logsData = [...state.logsData, ...result.data]; appendLogsRows(result.data); } - pagination.recordPage(result.data.length); + pagination.recordPage(result.data); } catch (err) { if (!isCurrent() || isAbortError(err)) return; // eslint-disable-next-line no-console @@ -453,6 +516,116 @@ async function loadMoreLogs() { } } +// Collapse toggle label helper +function updateCollapseToggleLabel() { + const btn = document.getElementById('chartCollapseToggle'); + if (!btn) return; + const chartSection = document.querySelector('.chart-section'); + const collapsed = chartSection?.classList.contains('chart-collapsed'); + btn.innerHTML = collapsed ? '▼ Show chart' : '▲ Hide chart'; + btn.title = collapsed ? 'Expand chart' : 'Collapse chart'; +} + +// Set up collapse toggle click handler +export function initChartCollapseToggle() { + const btn = document.getElementById('chartCollapseToggle'); + if (!btn) return; + btn.addEventListener('click', () => { + const chartSection = document.querySelector('.chart-section'); + if (!chartSection) return; + chartSection.classList.toggle('chart-collapsed'); + const collapsed = chartSection.classList.contains('chart-collapsed'); + localStorage.setItem('chartCollapsed', collapsed ? 'true' : 'false'); + updateCollapseToggleLabel(); + }); +} + +// Throttle helper +function throttle(fn, delay) { + let lastCall = 0; + let timer = null; + return (...args) => { + const now = Date.now(); + const remaining = delay - (now - lastCall); + if (remaining <= 0) { + if (timer) { + clearTimeout(timer); + timer = null; + } + lastCall = now; + fn(...args); + } else if (!timer) { + timer = setTimeout(() => { + lastCall = Date.now(); + timer = null; + fn(...args); + }, remaining); + } + }; +} + +// Scroll→Chart sync: update scrubber to match topmost visible log row +function syncScrubberToScroll() { + if (!state.showLogs || !state.logsData || state.logsData.length === 0) return; + + const container = logsView?.querySelector('.logs-table-container'); + if (!container) return; + + // Find the topmost visible row below the sticky chart + const chartSection = document.querySelector('.chart-section'); + const chartBottom = chartSection ? chartSection.getBoundingClientRect().bottom : 0; + + const rows = container.querySelectorAll('.logs-table tbody tr'); + let topRow = null; + for (const row of rows) { + const rect = row.getBoundingClientRect(); + if (rect.bottom > chartBottom) { + topRow = row; + break; + } + } + + if (!topRow || !topRow.dataset.rowIdx) return; + const rowIdx = parseInt(topRow.dataset.rowIdx, 10); + const rowData = state.logsData[rowIdx]; + if (!rowData || !rowData.timestamp) return; + + const timestamp = parseUTC(rowData.timestamp); + setScrubberPosition(timestamp); +} + +const throttledSyncScrubber = throttle(syncScrubberToScroll, 100); + +// Scroll log table to the row closest to a given timestamp +export function scrollLogsToTimestamp(timestamp) { + if (!state.showLogs || !state.logsData || state.logsData.length === 0) return; + + const targetMs = timestamp instanceof Date ? timestamp.getTime() : timestamp; + let closestIdx = 0; + let closestDiff = Infinity; + + for (let i = 0; i < state.logsData.length; i += 1) { + const row = state.logsData[i]; + if (!row.timestamp) { + closestIdx += 0; // eslint: no-continue workaround - skip rows without timestamp + } else { + const rowMs = parseUTC(row.timestamp).getTime(); + const diff = Math.abs(rowMs - targetMs); + if (diff < closestDiff) { + closestDiff = diff; + closestIdx = i; + } + } + } + + const container = logsView?.querySelector('.logs-table-container'); + if (!container) return; + const targetRow = container.querySelector(`tr[data-row-idx="${closestIdx}"]`); + if (targetRow) { + targetRow.scrollIntoView({ block: 'center', behavior: 'smooth' }); + } +} + function handleLogsScroll() { // Only handle scroll when logs view is visible if (!state.showLogs) return; @@ -466,6 +639,9 @@ function handleLogsScroll() { if (pagination.shouldTriggerLoad(scrollPercent, state.logsLoading)) { loadMoreLogs(); } + + // Sync chart scrubber to topmost visible log row + throttledSyncScrubber(); } export function setLogsElements(view, toggleBtn, filtersViewEl) { @@ -478,6 +654,9 @@ export function setLogsElements(view, toggleBtn, filtersViewEl) { // Set up click handler for copying row data setupLogRowClickHandler(); + + // Set up chart collapse toggle + initChartCollapseToggle(); } // Register callback for pinned column changes @@ -492,14 +671,23 @@ export function setOnShowFiltersView(callback) { export function toggleLogsView(saveStateToURL) { state.showLogs = !state.showLogs; + const dashboardContent = document.getElementById('dashboardContent'); if (state.showLogs) { logsView.classList.add('visible'); filtersView.classList.remove('visible'); viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Filters'; + dashboardContent.classList.add('logs-active'); + // Restore collapse state from localStorage + const chartSection = document.querySelector('.chart-section'); + if (chartSection && localStorage.getItem('chartCollapsed') === 'true') { + chartSection.classList.add('chart-collapsed'); + updateCollapseToggleLabel(); + } } else { logsView.classList.remove('visible'); filtersView.classList.add('visible'); viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Logs'; + dashboardContent.classList.remove('logs-active'); // Redraw chart after view becomes visible if (onShowFiltersView) { requestAnimationFrame(() => onShowFiltersView()); @@ -529,6 +717,7 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) const sql = await loadSql('logs', { database: DATABASE, table: getTable(), + columns: buildLogColumnsSql(state.pinnedColumns), timeFilter, hostFilter, facetFilters, @@ -542,7 +731,7 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) state.logsData = result.data; renderLogsTable(result.data); state.logsReady = true; - pagination.recordPage(result.data.length); + pagination.recordPage(result.data); } catch (err) { if (!isCurrent() || isAbortError(err)) return; // eslint-disable-next-line no-console diff --git a/js/pagination.js b/js/pagination.js index ab1954e..39efa75 100644 --- a/js/pagination.js +++ b/js/pagination.js @@ -14,21 +14,24 @@ export const PAGE_SIZE = 500; export class PaginationState { constructor(pageSize = PAGE_SIZE) { - this.offset = 0; + this.cursor = null; this.hasMore = true; this.loading = false; this.pageSize = pageSize; } reset() { - this.offset = 0; + this.cursor = null; this.hasMore = true; this.loading = false; } - recordPage(resultLength) { - this.offset += resultLength; + recordPage(rows) { + const resultLength = rows.length; this.hasMore = resultLength === this.pageSize; + if (resultLength > 0) { + this.cursor = rows[resultLength - 1].timestamp; + } } canLoadMore() { diff --git a/js/pagination.test.js b/js/pagination.test.js index f153aeb..ea233bf 100644 --- a/js/pagination.test.js +++ b/js/pagination.test.js @@ -22,7 +22,7 @@ describe('PaginationState', () => { describe('constructor', () => { it('initializes with default page size', () => { const ps = new PaginationState(); - assert.strictEqual(ps.offset, 0); + assert.strictEqual(ps.cursor, null); assert.strictEqual(ps.hasMore, true); assert.strictEqual(ps.loading, false); assert.strictEqual(ps.pageSize, PAGE_SIZE); @@ -35,22 +35,22 @@ describe('PaginationState', () => { }); describe('reset', () => { - it('resets offset, hasMore, and loading', () => { + it('resets cursor, hasMore, and loading', () => { const ps = new PaginationState(); - ps.offset = 250; + ps.cursor = '2025-01-15 10:30:00.123'; ps.hasMore = false; ps.loading = true; ps.reset(); - assert.strictEqual(ps.offset, 0); + assert.strictEqual(ps.cursor, null); assert.strictEqual(ps.hasMore, true); assert.strictEqual(ps.loading, false); }); it('preserves pageSize', () => { const ps = new PaginationState(100); - ps.offset = 50; + ps.cursor = '2025-01-15 10:30:00.123'; ps.reset(); @@ -60,45 +60,82 @@ describe('PaginationState', () => { describe('recordPage', () => { it('sets hasMore=true when result is a full page', () => { + const rows = Array.from({ length: PAGE_SIZE }, (_, i) => ({ + timestamp: `2025-01-15 10:30:00.${String(i).padStart(3, '0')}`, + })); const ps = new PaginationState(); - ps.recordPage(PAGE_SIZE); + ps.recordPage(rows); - assert.strictEqual(ps.offset, PAGE_SIZE); assert.strictEqual(ps.hasMore, true); }); it('sets hasMore=false when result is smaller than page size', () => { + const rows = [ + { timestamp: '2025-01-15 10:30:00.100' }, + { timestamp: '2025-01-15 10:30:00.050' }, + { timestamp: '2025-01-15 10:30:00.001' }, + ]; const ps = new PaginationState(); - ps.recordPage(123); + ps.recordPage(rows); - assert.strictEqual(ps.offset, 123); assert.strictEqual(ps.hasMore, false); }); it('sets hasMore=false when result is empty', () => { const ps = new PaginationState(); - ps.recordPage(0); + ps.recordPage([]); - assert.strictEqual(ps.offset, 0); assert.strictEqual(ps.hasMore, false); }); - it('accumulates offset across multiple pages', () => { + it('extracts cursor from last row timestamp', () => { + const rows = [ + { timestamp: '2025-01-15 10:30:00.300' }, + { timestamp: '2025-01-15 10:30:00.200' }, + { timestamp: '2025-01-15 10:30:00.100' }, + ]; const ps = new PaginationState(); - ps.recordPage(PAGE_SIZE); - ps.recordPage(PAGE_SIZE); - ps.recordPage(200); + ps.recordPage(rows); - assert.strictEqual(ps.offset, PAGE_SIZE * 2 + 200); + assert.strictEqual(ps.cursor, '2025-01-15 10:30:00.100'); + }); + + it('does not update cursor when result is empty', () => { + const ps = new PaginationState(); + ps.cursor = '2025-01-15 10:30:00.100'; + ps.recordPage([]); + + assert.strictEqual(ps.cursor, '2025-01-15 10:30:00.100'); + }); + + it('updates cursor across multiple pages', () => { + const ps = new PaginationState(2); + + ps.recordPage([ + { timestamp: '2025-01-15 10:30:00.300' }, + { timestamp: '2025-01-15 10:30:00.200' }, + ]); + assert.strictEqual(ps.cursor, '2025-01-15 10:30:00.200'); + assert.strictEqual(ps.hasMore, true); + + ps.recordPage([ + { timestamp: '2025-01-15 10:30:00.100' }, + ]); + assert.strictEqual(ps.cursor, '2025-01-15 10:30:00.100'); assert.strictEqual(ps.hasMore, false); }); it('uses custom page size for hasMore check', () => { - const ps = new PaginationState(10); - ps.recordPage(10); + const ps = new PaginationState(2); + ps.recordPage([ + { timestamp: '2025-01-15 10:30:00.200' }, + { timestamp: '2025-01-15 10:30:00.100' }, + ]); assert.strictEqual(ps.hasMore, true); - ps.recordPage(5); + ps.recordPage([ + { timestamp: '2025-01-15 10:30:00.050' }, + ]); assert.strictEqual(ps.hasMore, false); }); }); diff --git a/js/sql-loader.js b/js/sql-loader.js index 5e40c99..d20f0d4 100644 --- a/js/sql-loader.js +++ b/js/sql-loader.js @@ -73,6 +73,7 @@ const ALL_TEMPLATES = [ 'facet-search-pattern', 'investigate-facet', 'investigate-selection', + 'log-detail', ]; /** diff --git a/sql/queries/log-detail.sql b/sql/queries/log-detail.sql new file mode 100644 index 0000000..ceda620 --- /dev/null +++ b/sql/queries/log-detail.sql @@ -0,0 +1,4 @@ +SELECT * +FROM {{database}}.{{table}} +WHERE timestamp = toDateTime64('{{timestamp}}', 3) AND `request.host` = '{{host}}' +LIMIT 1 diff --git a/sql/queries/logs-more.sql b/sql/queries/logs-more.sql index 74a30fe..c5ed965 100644 --- a/sql/queries/logs-more.sql +++ b/sql/queries/logs-more.sql @@ -1,6 +1,5 @@ -SELECT * +SELECT {{columns}} FROM {{database}}.{{table}} -WHERE {{timeFilter}} {{hostFilter}} {{facetFilters}} {{additionalWhereClause}} +WHERE {{timeFilter}} AND timestamp < toDateTime64('{{cursor}}', 3) {{hostFilter}} {{facetFilters}} {{additionalWhereClause}} ORDER BY timestamp DESC LIMIT {{pageSize}} -OFFSET {{offset}} diff --git a/sql/queries/logs.sql b/sql/queries/logs.sql index 7815ee9..ac3e46d 100644 --- a/sql/queries/logs.sql +++ b/sql/queries/logs.sql @@ -1,4 +1,4 @@ -SELECT * +SELECT {{columns}} FROM {{database}}.{{table}} WHERE {{timeFilter}} {{hostFilter}} {{facetFilters}} {{additionalWhereClause}} ORDER BY timestamp DESC From a5eb5a5f25234888432e0b2ee179dc26373289a5 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 12 Feb 2026 11:55:52 +0100 Subject: [PATCH 02/28] fix: address PR review feedback for logs view improvements - Add timestamp format validation before SQL interpolation - Validate pinned column names against allowed pattern - Fix throttle stale args bug - Replace no-op with proper continue statement - Initialize collapse toggle label on page load - Add aria-hidden to toggle button arrow entities - Guard against null cursor in canLoadMore() Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- dashboard.html | 2 +- js/columns.js | 5 ++++- js/logs.js | 42 +++++++++++++++++++++++++++++++++--------- js/pagination.js | 2 +- js/pagination.test.js | 9 ++++++++- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/dashboard.html b/dashboard.html index b02aced..3e94b0d 100644 --- a/dashboard.html +++ b/dashboard.html @@ -80,7 +80,7 @@

Requests over time

- + diff --git a/js/columns.js b/js/columns.js index 524d7c1..ab51e3f 100644 --- a/js/columns.js +++ b/js/columns.js @@ -194,10 +194,13 @@ const ALWAYS_NEEDED_COLUMNS = ['timestamp', 'source', 'sample_hash']; * @param {string[]} [pinnedColumns] - Additional pinned columns to include. * @returns {string} Comma-separated, backtick-quoted column list for SQL SELECT. */ +const VALID_COLUMN_RE = /^[a-z][a-z0-9_.]*$/i; + export function buildLogColumnsSql(pinnedColumns = []) { const seen = new Set(); const cols = []; - for (const col of [...ALWAYS_NEEDED_COLUMNS, ...LOG_COLUMN_ORDER, ...pinnedColumns]) { + const safePinned = pinnedColumns.filter((col) => VALID_COLUMN_RE.test(col)); + for (const col of [...ALWAYS_NEEDED_COLUMNS, ...LOG_COLUMN_ORDER, ...safePinned]) { if (!seen.has(col)) { seen.add(col); cols.push(`\`${col}\``); diff --git a/js/logs.js b/js/logs.js index 6b75bd2..2d370f3 100644 --- a/js/logs.js +++ b/js/logs.js @@ -25,6 +25,8 @@ import { PAGE_SIZE, PaginationState } from './pagination.js'; import { setScrubberPosition } from './chart.js'; import { parseUTC } from './chart-state.js'; +const TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/; + /** * Build ordered log column list from available columns. * @param {string[]} allColumns @@ -260,11 +262,17 @@ function showDetailLoading() { */ async function fetchFullRow(partialRow) { const { timestamp } = partialRow; + const tsStr = String(timestamp); + if (!TIMESTAMP_RE.test(tsStr)) { + // eslint-disable-next-line no-console + console.warn('fetchFullRow: invalid timestamp format, aborting', tsStr); + return null; + } const host = partialRow['request.host'] || ''; const sql = await loadSql('log-detail', { database: DATABASE, table: getTable(), - timestamp: String(timestamp), + timestamp: tsStr, host: host.replace(/'/g, "\\'"), }); const result = await query(sql); @@ -476,6 +484,14 @@ export function renderLogsTable(data) { async function loadMoreLogs() { if (!pagination.canLoadMore()) return; + + // Validate cursor format before interpolating into SQL + if (!TIMESTAMP_RE.test(pagination.cursor)) { + // eslint-disable-next-line no-console + console.warn('loadMoreLogs: invalid cursor format, aborting', pagination.cursor); + return; + } + pagination.loading = true; const requestContext = getRequestContext('dashboard'); const { requestId, signal, scope } = requestContext; @@ -522,7 +538,7 @@ function updateCollapseToggleLabel() { if (!btn) return; const chartSection = document.querySelector('.chart-section'); const collapsed = chartSection?.classList.contains('chart-collapsed'); - btn.innerHTML = collapsed ? '▼ Show chart' : '▲ Hide chart'; + btn.innerHTML = collapsed ? ' Show chart' : ' Hide chart'; btn.title = collapsed ? 'Expand chart' : 'Collapse chart'; } @@ -538,12 +554,14 @@ export function initChartCollapseToggle() { localStorage.setItem('chartCollapsed', collapsed ? 'true' : 'false'); updateCollapseToggleLabel(); }); + updateCollapseToggleLabel(); } // Throttle helper function throttle(fn, delay) { let lastCall = 0; let timer = null; + let pendingArgs = null; return (...args) => { const now = Date.now(); const remaining = delay - (now - lastCall); @@ -552,14 +570,20 @@ function throttle(fn, delay) { clearTimeout(timer); timer = null; } + pendingArgs = null; lastCall = now; fn(...args); - } else if (!timer) { - timer = setTimeout(() => { - lastCall = Date.now(); - timer = null; - fn(...args); - }, remaining); + } else { + pendingArgs = args; + if (!timer) { + timer = setTimeout(() => { + lastCall = Date.now(); + timer = null; + const latestArgs = pendingArgs; + pendingArgs = null; + fn(...latestArgs); + }, remaining); + } } }; } @@ -607,7 +631,7 @@ export function scrollLogsToTimestamp(timestamp) { for (let i = 0; i < state.logsData.length; i += 1) { const row = state.logsData[i]; if (!row.timestamp) { - closestIdx += 0; // eslint: no-continue workaround - skip rows without timestamp + continue; // eslint-disable-line no-continue } else { const rowMs = parseUTC(row.timestamp).getTime(); const diff = Math.abs(rowMs - targetMs); diff --git a/js/pagination.js b/js/pagination.js index 39efa75..70d24dd 100644 --- a/js/pagination.js +++ b/js/pagination.js @@ -35,7 +35,7 @@ export class PaginationState { } canLoadMore() { - return this.hasMore && !this.loading; + return this.hasMore && !this.loading && this.cursor != null; } shouldTriggerLoad(scrollPercent, globalLoading) { diff --git a/js/pagination.test.js b/js/pagination.test.js index ea233bf..611aca7 100644 --- a/js/pagination.test.js +++ b/js/pagination.test.js @@ -141,11 +141,17 @@ describe('PaginationState', () => { }); describe('canLoadMore', () => { - it('returns true when hasMore and not loading', () => { + it('returns true when hasMore, not loading, and cursor is set', () => { const ps = new PaginationState(); + ps.cursor = '2025-01-15 10:30:00.000'; assert.strictEqual(ps.canLoadMore(), true); }); + it('returns false when cursor is null', () => { + const ps = new PaginationState(); + assert.strictEqual(ps.canLoadMore(), false); + }); + it('returns false when loading', () => { const ps = new PaginationState(); ps.loading = true; @@ -169,6 +175,7 @@ describe('PaginationState', () => { describe('shouldTriggerLoad', () => { it('triggers when scrolled past 50% and can load more', () => { const ps = new PaginationState(); + ps.cursor = '2025-01-15 10:30:00.000'; assert.strictEqual(ps.shouldTriggerLoad(0.6, false), true); }); From 53eb32e3fcf6ce71794a6ad39383ae4d3b426156 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 12 Feb 2026 17:58:14 +0100 Subject: [PATCH 03/28] feat: integrate VirtualTable into logs view for virtual scrolling Replace innerHTML-based rendering with VirtualTable component: - js/logs.js: rewrite to use VirtualTable with getData callback, cursor-based page cache, and onVisibleRangeChange for chart sync - js/virtual-table.js: add new VirtualTable component (from Wave 1) - js/virtual-table.test.js: add VirtualTable unit tests (from Wave 1) - css/logs.css: add container height and loading-row styles - sql/queries/logs-at.sql: new template for jump-to-timestamp queries - js/sql-loader.js: register logs-at template in preload list Removed: renderLogsTable, appendLogsRows, handleLogsScroll, loadMoreLogs, syncScrubberToScroll, setupLogRowClickHandler, updatePinnedOffsets, getApproxPinnedOffsets, window scroll listener. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 8 + js/logs.js | 530 ++++++++++++++++----------------------- js/sql-loader.js | 1 + js/virtual-table.js | 322 ++++++++++++++++++++++++ js/virtual-table.test.js | 371 +++++++++++++++++++++++++++ sql/queries/logs-at.sql | 5 + 6 files changed, 923 insertions(+), 314 deletions(-) create mode 100644 js/virtual-table.js create mode 100644 js/virtual-table.test.js create mode 100644 sql/queries/logs-at.sql diff --git a/css/logs.css b/css/logs.css index 37e5a7a..80286cd 100644 --- a/css/logs.css +++ b/css/logs.css @@ -25,6 +25,8 @@ border: 1px solid var(--border); overflow-x: auto; transition: filter 0.2s ease-out; + height: calc(100vh - 200px); + min-height: 400px; } @media (max-width: 600px) { @@ -175,6 +177,12 @@ color: var(--text-secondary); } +/* Virtual table loading placeholder rows */ +.logs-table .loading-row td { + background: var(--card-bg); + color: transparent; +} + /* Copy feedback toast */ .copy-feedback { position: fixed; diff --git a/js/logs.js b/js/logs.js index 2d370f3..0934f0d 100644 --- a/js/logs.js +++ b/js/logs.js @@ -20,10 +20,11 @@ import { getColorForColumn } from './colors/index.js'; import { getRequestContext, isRequestCurrent } from './request-context.js'; import { LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, buildLogColumnsSql } from './columns.js'; import { loadSql } from './sql-loader.js'; -import { buildLogRowHtml, buildLogTableHeaderHtml } from './templates/logs-table.js'; -import { PAGE_SIZE, PaginationState } from './pagination.js'; +import { formatLogCell } from './templates/logs-table.js'; +import { PAGE_SIZE } from './pagination.js'; import { setScrubberPosition } from './chart.js'; import { parseUTC } from './chart-state.js'; +import { VirtualTable } from './virtual-table.js'; const TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/; @@ -41,56 +42,19 @@ function getLogColumns(allColumns) { } /** - * Build approximate left offsets for pinned columns. - * @param {string[]} pinned - * @param {number} width - * @returns {Record} + * Build VirtualTable column descriptors from column name list. + * @param {string[]} columns - ordered column names + * @returns {Array<{key:string, label:string, pinned?:boolean, width?:number}>} */ -function getApproxPinnedOffsets(pinned, width) { - const offsets = {}; - pinned.forEach((col, index) => { - offsets[col] = index * width; - }); - return offsets; -} - -/** - * Update pinned column offsets based on actual column widths. - * @param {HTMLElement} container - * @param {string[]} pinned - */ -function updatePinnedOffsets(container, pinned) { - if (pinned.length === 0) return; - - requestAnimationFrame(() => { - const table = container.querySelector('.logs-table'); - if (!table) return; - const headerCells = table.querySelectorAll('thead th'); - const pinnedWidths = []; - let cumLeft = 0; - - for (let i = 0; i < pinned.length; i += 1) { - pinnedWidths.push(cumLeft); - cumLeft += headerCells[i].offsetWidth; - } - - headerCells.forEach((headerCell, idx) => { - if (idx < pinned.length) { - const th = headerCell; - th.style.left = `${pinnedWidths[idx]}px`; - } - }); - - const rows = table.querySelectorAll('tbody tr'); - rows.forEach((row) => { - const cells = row.querySelectorAll('td'); - cells.forEach((cell, idx) => { - if (idx < pinned.length) { - const td = cell; - td.style.left = `${pinnedWidths[idx]}px`; - } - }); - }); +function buildVirtualColumns(columns) { + const pinned = state.pinnedColumns; + return columns.map((col) => { + const isPinned = pinned.includes(col); + const label = LOG_COLUMN_SHORT_LABELS[col] || col; + const entry = { key: col, label }; + if (isPinned) entry.pinned = true; + if (col === 'timestamp') entry.width = 180; + return entry; }); } @@ -99,8 +63,13 @@ let logsView = null; let viewToggleBtn = null; let filtersView = null; -// Pagination state -const pagination = new PaginationState(); +// VirtualTable instance +let virtualTable = null; + +// Page cache for cursor-based pagination +// Maps pageIndex → { rows, cursor (timestamp of last row) } +const pageCache = new Map(); +let currentColumns = []; // Show brief "Copied!" feedback function showCopyFeedback() { @@ -308,13 +277,26 @@ function initLogDetailModal() { } } +/** + * Look up a row from the page cache by virtual index. + * @param {number} rowIdx + * @returns {Object|null} + */ +function getRowFromCache(rowIdx) { + const pageIdx = Math.floor(rowIdx / PAGE_SIZE); + const page = pageCache.get(pageIdx); + if (!page) return null; + const offset = rowIdx - pageIdx * PAGE_SIZE; + return offset < page.rows.length ? page.rows[offset] : null; +} + /** * Open log detail modal for a row. * Fetches full row data on demand if not already present. * @param {number} rowIdx + * @param {Object} row */ -export async function openLogDetailModal(rowIdx) { - const row = state.logsData[rowIdx]; +export async function openLogDetailModal(rowIdx, row) { if (!row) return; initLogDetailModal(); @@ -324,17 +306,9 @@ export async function openLogDetailModal(rowIdx) { showDetailLoading(); logDetailModal.showModal(); - // Check if row already has full data (e.g. from a previous fetch) - if (row.fullRowData) { - renderLogDetailContent(row.fullRowData); - return; - } - try { const fullRow = await fetchFullRow(row); if (fullRow) { - // Cache the full row for future opens - row.fullRowData = fullRow; renderLogDetailContent(fullRow); } else { // Fallback: render with partial data @@ -350,7 +324,7 @@ export async function openLogDetailModal(rowIdx) { // Copy row data as JSON when clicking on row background export function copyLogRow(rowIdx) { - const row = state.logsData[rowIdx]; + const row = getRowFromCache(rowIdx); if (!row) return; // Convert flat dot notation to nested object @@ -380,158 +354,11 @@ export function copyLogRow(rowIdx) { }); } -// Set up click handler for row background clicks -export function setupLogRowClickHandler() { - const container = logsView?.querySelector('.logs-table-container'); - if (!container) return; - - container.addEventListener('click', (e) => { - // Only handle clicks directly on td or tr (not on links, buttons, or spans) - const { target } = e; - if (target.tagName !== 'TD' && target.tagName !== 'TR') return; - - // Don't open modal if clicking on a clickable cell (filter action) - if (target.classList.contains('clickable')) return; - - // Find the row - const row = target.closest('tr'); - if (!row || !row.dataset.rowIdx) return; - - const rowIdx = parseInt(row.dataset.rowIdx, 10); - openLogDetailModal(rowIdx); - }); -} - function renderLogsError(message) { const container = logsView.querySelector('.logs-table-container'); container.innerHTML = `
Error loading logs: ${escapeHtml(message)}
`; } -// Append rows to existing logs table (for infinite scroll) -function appendLogsRows(data) { - const container = logsView.querySelector('.logs-table-container'); - const tbody = container.querySelector('.logs-table tbody'); - if (!tbody || data.length === 0) return; - - // Get columns from existing table header - const headerCells = container.querySelectorAll('.logs-table thead th'); - const columns = Array.from(headerCells).map((th) => th.title || th.textContent); - - // Map short names back to full names - const shortToFull = Object.fromEntries( - Object.entries(LOG_COLUMN_SHORT_LABELS).map(([full, short]) => [short, full]), - ); - - const fullColumns = columns.map((col) => shortToFull[col] || col); - const pinned = state.pinnedColumns.filter((col) => fullColumns.includes(col)); - - // Get starting index from existing rows - const existingRows = tbody.querySelectorAll('tr').length; - - let html = ''; - for (let i = 0; i < data.length; i += 1) { - const rowIdx = existingRows + i; - html += buildLogRowHtml({ - row: data[i], columns: fullColumns, rowIdx, pinned, - }); - } - - tbody.insertAdjacentHTML('beforeend', html); - - updatePinnedOffsets(container, pinned); -} - -export function renderLogsTable(data) { - const container = logsView.querySelector('.logs-table-container'); - - if (data.length === 0) { - container.innerHTML = '
No logs matching current filters
'; - return; - } - - // Get all column names from first row - const allColumns = Object.keys(data[0]); - - // Sort columns: pinned first, then preferred order, then the rest - const pinned = state.pinnedColumns.filter((col) => allColumns.includes(col)); - const columns = getLogColumns(allColumns); - - // Calculate left offsets for sticky pinned columns - const COL_WIDTH = 120; - const pinnedOffsets = getApproxPinnedOffsets(pinned, COL_WIDTH); - - let html = ` - - - - ${buildLogTableHeaderHtml(columns, pinned, pinnedOffsets)} - - - - `; - - for (let rowIdx = 0; rowIdx < data.length; rowIdx += 1) { - html += buildLogRowHtml({ - row: data[rowIdx], columns, rowIdx, pinned, pinnedOffsets, - }); - } - - html += '
'; - container.innerHTML = html; - - updatePinnedOffsets(container, pinned); -} - -async function loadMoreLogs() { - if (!pagination.canLoadMore()) return; - - // Validate cursor format before interpolating into SQL - if (!TIMESTAMP_RE.test(pagination.cursor)) { - // eslint-disable-next-line no-console - console.warn('loadMoreLogs: invalid cursor format, aborting', pagination.cursor); - return; - } - - pagination.loading = true; - const requestContext = getRequestContext('dashboard'); - const { requestId, signal, scope } = requestContext; - const isCurrent = () => isRequestCurrent(requestId, scope); - - const timeFilter = getTimeFilter(); - const hostFilter = getHostFilter(); - const facetFilters = getFacetFilters(); - - const sql = await loadSql('logs-more', { - database: DATABASE, - table: getTable(), - columns: buildLogColumnsSql(state.pinnedColumns), - timeFilter, - hostFilter, - facetFilters, - additionalWhereClause: state.additionalWhereClause, - pageSize: String(PAGE_SIZE), - cursor: pagination.cursor, - }); - - try { - const result = await query(sql, { signal }); - if (!isCurrent()) return; - if (result.data.length > 0) { - state.logsData = [...state.logsData, ...result.data]; - appendLogsRows(result.data); - } - pagination.recordPage(result.data); - } catch (err) { - if (!isCurrent() || isAbortError(err)) return; - // eslint-disable-next-line no-console - console.error('Load more logs error:', err); - } finally { - if (isCurrent()) { - pagination.loading = false; - } - } -} - // Collapse toggle label helper function updateCollapseToggleLabel() { const btn = document.getElementById('chartCollapseToggle'); @@ -557,115 +384,150 @@ export function initChartCollapseToggle() { updateCollapseToggleLabel(); } -// Throttle helper -function throttle(fn, delay) { - let lastCall = 0; - let timer = null; - let pendingArgs = null; - return (...args) => { - const now = Date.now(); - const remaining = delay - (now - lastCall); - if (remaining <= 0) { - if (timer) { - clearTimeout(timer); - timer = null; - } - pendingArgs = null; - lastCall = now; - fn(...args); - } else { - pendingArgs = args; - if (!timer) { - timer = setTimeout(() => { - lastCall = Date.now(); - timer = null; - const latestArgs = pendingArgs; - pendingArgs = null; - fn(...latestArgs); - }, remaining); - } - } - }; +/** + * renderCell callback for VirtualTable. + * Returns HTML string for a single cell. + */ +function renderCell(col, value) { + const { displayValue, cellClass, colorIndicator } = formatLogCell(col.key, value); + const escaped = escapeHtml(displayValue); + const cls = cellClass ? ` class="${cellClass}"` : ''; + return `${colorIndicator}${escaped}`; } -// Scroll→Chart sync: update scrubber to match topmost visible log row -function syncScrubberToScroll() { - if (!state.showLogs || !state.logsData || state.logsData.length === 0) return; - - const container = logsView?.querySelector('.logs-table-container'); - if (!container) return; - - // Find the topmost visible row below the sticky chart - const chartSection = document.querySelector('.chart-section'); - const chartBottom = chartSection ? chartSection.getBoundingClientRect().bottom : 0; - - const rows = container.querySelectorAll('.logs-table tbody tr'); - let topRow = null; - for (const row of rows) { - const rect = row.getBoundingClientRect(); - if (rect.bottom > chartBottom) { - topRow = row; - break; - } +/** + * getData callback for VirtualTable. + * Fetches a page of log rows from ClickHouse using cursor-based pagination. + */ +async function getData(startIdx, count) { + const pageIdx = Math.floor(startIdx / PAGE_SIZE); + + // Return from cache if available + if (pageCache.has(pageIdx)) { + const page = pageCache.get(pageIdx); + const offset = startIdx - pageIdx * PAGE_SIZE; + return page.rows.slice(offset, offset + count); } - if (!topRow || !topRow.dataset.rowIdx) return; - const rowIdx = parseInt(topRow.dataset.rowIdx, 10); - const rowData = state.logsData[rowIdx]; - if (!rowData || !rowData.timestamp) return; + const timeFilter = getTimeFilter(); + const hostFilter = getHostFilter(); + const facetFilters = getFacetFilters(); + const sqlParams = { + database: DATABASE, + table: getTable(), + columns: buildLogColumnsSql(state.pinnedColumns), + timeFilter, + hostFilter, + facetFilters, + additionalWhereClause: state.additionalWhereClause, + pageSize: String(PAGE_SIZE), + }; - const timestamp = parseUTC(rowData.timestamp); - setScrubberPosition(timestamp); -} + let sql; + if (pageIdx === 0) { + // Initial page — no cursor needed + sql = await loadSql('logs', sqlParams); + } else { + // Find the cursor from the nearest previous cached page + let cursor = null; + for (let p = pageIdx - 1; p >= 0; p -= 1) { + const prev = pageCache.get(p); + if (prev && prev.cursor) { + cursor = prev.cursor; + break; + } + } -const throttledSyncScrubber = throttle(syncScrubberToScroll, 100); + if (cursor && TIMESTAMP_RE.test(cursor)) { + sql = await loadSql('logs-more', { ...sqlParams, cursor }); + } else { + // No cursor available — fall back to initial query + sql = await loadSql('logs', sqlParams); + } + } -// Scroll log table to the row closest to a given timestamp -export function scrollLogsToTimestamp(timestamp) { - if (!state.showLogs || !state.logsData || state.logsData.length === 0) return; + try { + const result = await query(sql); + const rows = result.data; + const cursor = rows.length > 0 ? rows[rows.length - 1].timestamp : null; + pageCache.set(pageIdx, { rows, cursor }); + + // Update columns on first data load + if (rows.length > 0 && currentColumns.length === 0) { + currentColumns = getLogColumns(Object.keys(rows[0])); + if (virtualTable) { + virtualTable.setColumns(buildVirtualColumns(currentColumns)); + } + } - const targetMs = timestamp instanceof Date ? timestamp.getTime() : timestamp; - let closestIdx = 0; - let closestDiff = Infinity; + // Also update logsData on state for backwards compat with detail modal + if (pageIdx === 0) { + state.logsData = rows; + } - for (let i = 0; i < state.logsData.length; i += 1) { - const row = state.logsData[i]; - if (!row.timestamp) { - continue; // eslint-disable-line no-continue - } else { - const rowMs = parseUTC(row.timestamp).getTime(); - const diff = Math.abs(rowMs - targetMs); - if (diff < closestDiff) { - closestDiff = diff; - closestIdx = i; - } + const offset = startIdx - pageIdx * PAGE_SIZE; + return rows.slice(offset, offset + count); + } catch (err) { + if (!isAbortError(err)) { + // eslint-disable-next-line no-console + console.error('getData error:', err); } + return []; } +} - const container = logsView?.querySelector('.logs-table-container'); - if (!container) return; - const targetRow = container.querySelector(`tr[data-row-idx="${closestIdx}"]`); - if (targetRow) { - targetRow.scrollIntoView({ block: 'center', behavior: 'smooth' }); +/** + * Destroy the current virtual table if it exists. + */ +function destroyVirtualTable() { + if (virtualTable) { + virtualTable.destroy(); + virtualTable = null; } } -function handleLogsScroll() { - // Only handle scroll when logs view is visible - if (!state.showLogs) return; - - const { scrollHeight } = document.documentElement; - const scrollTop = window.scrollY; - const clientHeight = window.innerHeight; +/** + * Create or reconfigure the VirtualTable instance. + */ +function ensureVirtualTable() { + const container = logsView.querySelector('.logs-table-container'); + if (!container) return; - // Load more when scrolled to last 50% - const scrollPercent = (scrollTop + clientHeight) / scrollHeight; - if (pagination.shouldTriggerLoad(scrollPercent, state.logsLoading)) { - loadMoreLogs(); + if (virtualTable) { + // Already exists — just clear cache and re-render + virtualTable.clearCache(); + virtualTable.setTotalRows(0); + return; } - // Sync chart scrubber to topmost visible log row - throttledSyncScrubber(); + // Clear loading placeholder + container.innerHTML = ''; + + virtualTable = new VirtualTable({ + container, + rowHeight: 28, + columns: currentColumns.length > 0 ? buildVirtualColumns(currentColumns) : [], + getData, + renderCell, + onVisibleRangeChange(firstRow, lastRow) { + // Sync chart scrubber to the middle visible row + const midIdx = Math.floor((firstRow + lastRow) / 2); + const row = getRowFromCache(midIdx); + if (row && row.timestamp) { + setScrubberPosition(parseUTC(row.timestamp)); + } + }, + onRowClick(idx, row) { + openLogDetailModal(idx, row); + }, + }); +} + +// Scroll log table to the row closest to a given timestamp +export function scrollLogsToTimestamp(timestamp) { + if (!state.showLogs || !virtualTable) return; + const targetMs = timestamp instanceof Date ? timestamp.getTime() : timestamp; + virtualTable.scrollToTimestamp(targetMs, (row) => parseUTC(row.timestamp).getTime()); } export function setLogsElements(view, toggleBtn, filtersViewEl) { @@ -673,18 +535,22 @@ export function setLogsElements(view, toggleBtn, filtersViewEl) { viewToggleBtn = toggleBtn; filtersView = filtersViewEl; - // Set up scroll listener for infinite scroll on window - window.addEventListener('scroll', handleLogsScroll); - - // Set up click handler for copying row data - setupLogRowClickHandler(); - // Set up chart collapse toggle initChartCollapseToggle(); } // Register callback for pinned column changes -setOnPinnedColumnsChange(renderLogsTable); +setOnPinnedColumnsChange(() => { + if (!virtualTable || currentColumns.length === 0) return; + + // Rebuild column list with new pinned state + // Re-derive from current data keys if available + const page0 = pageCache.get(0); + if (page0 && page0.rows.length > 0) { + currentColumns = getLogColumns(Object.keys(page0.rows[0])); + } + virtualTable.setColumns(buildVirtualColumns(currentColumns)); +}); // Callback for redrawing chart when switching views let onShowFiltersView = null; @@ -716,10 +582,21 @@ export function toggleLogsView(saveStateToURL) { if (onShowFiltersView) { requestAnimationFrame(() => onShowFiltersView()); } + // Clean up virtual table when leaving logs view + destroyVirtualTable(); } saveStateToURL(); } +/** + * Estimate total rows from chart data bucket counts. + * @returns {number} + */ +function estimateTotalRows() { + if (!state.chartData || !state.chartData.buckets) return 0; + return state.chartData.buckets.reduce((sum, b) => sum + (b.total || b.count || 0), 0); +} + export async function loadLogs(requestContext = getRequestContext('dashboard')) { const { requestId, signal, scope } = requestContext; const isCurrent = () => isRequestCurrent(requestId, scope); @@ -727,13 +604,17 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) state.logsLoading = true; state.logsReady = false; - // Reset pagination state - pagination.reset(); + // Reset page cache + pageCache.clear(); + currentColumns = []; // Apply blur effect while loading const container = logsView.querySelector('.logs-table-container'); container.classList.add('updating'); + // Set up virtual table + ensureVirtualTable(); + const timeFilter = getTimeFilter(); const hostFilter = getHostFilter(); const facetFilters = getFacetFilters(); @@ -752,10 +633,31 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) try { const result = await query(sql, { signal }); if (!isCurrent()) return; - state.logsData = result.data; - renderLogsTable(result.data); + + const rows = result.data; + state.logsData = rows; state.logsReady = true; - pagination.recordPage(result.data); + + if (rows.length === 0) { + container.innerHTML = '
No logs matching current filters
'; + destroyVirtualTable(); + return; + } + + // Populate initial page cache + const cursor = rows.length > 0 ? rows[rows.length - 1].timestamp : null; + pageCache.set(0, { rows, cursor }); + + // Set columns from data + currentColumns = getLogColumns(Object.keys(rows[0])); + if (virtualTable) { + virtualTable.setColumns(buildVirtualColumns(currentColumns)); + + // Estimate total rows: use chart data if available, otherwise extrapolate + const estimated = estimateTotalRows(); + const total = estimated > rows.length ? estimated : rows.length * 10; + virtualTable.setTotalRows(total); + } } catch (err) { if (!isCurrent() || isAbortError(err)) return; // eslint-disable-next-line no-console diff --git a/js/sql-loader.js b/js/sql-loader.js index d20f0d4..79e9990 100644 --- a/js/sql-loader.js +++ b/js/sql-loader.js @@ -62,6 +62,7 @@ const ALL_TEMPLATES = [ 'time-series', 'logs', 'logs-more', + 'logs-at', 'breakdown', 'breakdown-facet', 'breakdown-missing', diff --git a/js/virtual-table.js b/js/virtual-table.js new file mode 100644 index 0000000..81bf029 --- /dev/null +++ b/js/virtual-table.js @@ -0,0 +1,322 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +const DEFAULT_ROW_HEIGHT = 28; +const DEFAULT_OVERSCAN = 10; +const MAX_CACHED_PAGES = 20; + +function getPinnedOffsets(columns) { + const offsets = new Map(); + let left = 0; + for (const col of columns) { + if (col.pinned) { + offsets.set(col.key, left); + left += col.width || 120; + } + } + return offsets; +} + +function makeCellStyle(col, offsets) { + if (!col.pinned) return ''; + const left = offsets.get(col.key); + return ` style="position:sticky;left:${left}px;z-index:1"`; +} + +function findInCache(cache, idx) { + for (const page of cache.values()) { + const offset = idx - page.startIdx; + if (offset >= 0 && offset < page.rows.length) { + return page.rows[offset]; + } + } + return null; +} + +/** + * Minimal virtual-scrolling table. Renders only visible rows into a + * standard HTML , backed by an async getData callback. + */ +export class VirtualTable { + /** + * @param {Object} opts + * @param {HTMLElement} opts.container + * @param {number} [opts.rowHeight=28] + * @param {Array<{key:string, label:string, pinned?:boolean, width?:number}>} opts.columns + * @param {(start:number, count:number) => Promise} opts.getData + * @param {(col:Object, value:unknown, row:Object) => string} opts.renderCell + * @param {(first:number, last:number) => void} [opts.onVisibleRangeChange] + * @param {(idx:number, row:Object) => void} [opts.onRowClick] + */ + constructor({ + container, rowHeight, columns, getData, renderCell, + onVisibleRangeChange, onRowClick, + }) { + this.container = container; + this.rowHeight = rowHeight || DEFAULT_ROW_HEIGHT; + this.columns = columns; + this.getDataFn = getData; + this.renderCellFn = renderCell; + this.onRangeChange = onVisibleRangeChange || null; + this.onRowClickFn = onRowClick || null; + + this.totalRows = 0; + this.cache = new Map(); + this.pending = new Set(); + this.rafId = null; + this.lastRange = null; + + this.initDom(); + this.initEvents(); + } + + initDom() { + this.container.style.overflowY = 'auto'; + this.container.style.position = 'relative'; + + this.spacer = document.createElement('div'); + this.spacer.style.width = '1px'; + this.spacer.style.height = '0px'; + this.spacer.style.pointerEvents = 'none'; + + this.table = document.createElement('table'); + this.table.className = 'logs-table'; + this.table.style.position = 'sticky'; + this.table.style.top = '0'; + + this.thead = document.createElement('thead'); + this.tbody = document.createElement('tbody'); + this.table.appendChild(this.thead); + this.table.appendChild(this.tbody); + + this.container.appendChild(this.table); + this.container.appendChild(this.spacer); + + this.updateHeader(); + } + + updateHeader() { + const tr = document.createElement('tr'); + let pinnedLeft = 0; + for (const col of this.columns) { + const th = document.createElement('th'); + th.textContent = col.label || col.key; + th.title = col.key; + if (col.width) th.style.width = `${col.width}px`; + if (col.pinned) { + th.style.position = 'sticky'; + th.style.left = `${pinnedLeft}px`; + th.style.zIndex = '2'; + pinnedLeft += col.width || 120; + } + tr.appendChild(th); + } + this.thead.innerHTML = ''; + this.thead.appendChild(tr); + } + + initEvents() { + this.scrollHandler = () => { + if (this.rafId) return; + this.rafId = requestAnimationFrame(() => { + this.rafId = null; + this.renderRows(); + }); + }; + this.container.addEventListener( + 'scroll', + this.scrollHandler, + { passive: true }, + ); + + if (this.onRowClickFn) { + this.clickHandler = (e) => { + const tr = e.target.closest('tr[data-row-idx]'); + if (!tr) return; + const idx = parseInt(tr.dataset.rowIdx, 10); + const row = findInCache(this.cache, idx); + if (row) this.onRowClickFn(idx, row); + }; + this.tbody.addEventListener('click', this.clickHandler); + } + } + + computeRange() { + const { scrollTop } = this.container; + const viewHeight = this.container.clientHeight; + const start = Math.max( + 0, + Math.floor(scrollTop / this.rowHeight) - DEFAULT_OVERSCAN, + ); + const visible = Math.ceil(viewHeight / this.rowHeight); + const end = Math.min( + this.totalRows, + start + visible + DEFAULT_OVERSCAN * 2, + ); + return { start, end }; + } + + renderRows() { + if (this.totalRows === 0) { + this.tbody.innerHTML = ''; + this.lastRange = null; + return; + } + + const { start, end } = this.computeRange(); + + if ( + this.lastRange + && this.lastRange.start === start + && this.lastRange.end === end + ) return; + this.lastRange = { start, end }; + + const offsets = getPinnedOffsets(this.columns); + let html = ''; + let fetchStart = -1; + + for (let i = start; i < end; i += 1) { + const row = findInCache(this.cache, i); + const top = i * this.rowHeight; + + if (row) { + html += ``; + for (const col of this.columns) { + const sty = makeCellStyle(col, offsets); + html += `${this.renderCellFn(col, row[col.key], row)}`; + } + html += ''; + } else { + html += ``; + for (const col of this.columns) { + html += ` `; + } + html += ''; + if (fetchStart === -1) fetchStart = i; + } + } + + this.tbody.style.position = 'relative'; + this.tbody.style.height = `${this.totalRows * this.rowHeight}px`; + this.tbody.innerHTML = html; + + if (fetchStart !== -1) { + this.fetchRange(fetchStart, end); + } + + if (this.onRangeChange) { + const visStart = Math.max(start + DEFAULT_OVERSCAN, 0); + const visEnd = Math.min(end - DEFAULT_OVERSCAN, this.totalRows); + this.onRangeChange(visStart, visEnd); + } + } + + async fetchRange(startIdx, endIdx) { + const count = endIdx - startIdx; + if (this.pending.has(startIdx)) return; + this.pending.add(startIdx); + + try { + const rows = await this.getDataFn(startIdx, count); + this.cache.set(startIdx, { startIdx, rows }); + this.evictDistantPages(); + this.lastRange = null; + this.renderRows(); + } finally { + this.pending.delete(startIdx); + } + } + + evictDistantPages() { + if (this.cache.size <= MAX_CACHED_PAGES) return; + const center = this.container.scrollTop / this.rowHeight; + const sorted = [...this.cache.entries()] + .map(([key, page]) => ({ + key, + dist: Math.abs(page.startIdx - center), + })) + .sort((a, b) => b.dist - a.dist); + + while (this.cache.size > MAX_CACHED_PAGES) { + this.cache.delete(sorted.shift().key); + } + } + + /* ---- Public API ---- */ + + setTotalRows(n) { + this.totalRows = n; + this.spacer.style.height = `${n * this.rowHeight}px`; + this.lastRange = null; + this.renderRows(); + } + + setColumns(cols) { + this.columns = cols; + this.updateHeader(); + this.lastRange = null; + this.renderRows(); + } + + scrollToRow(index) { + this.container.scrollTop = Math.max(0, index * this.rowHeight); + } + + scrollToTimestamp(ts, getTimestamp) { + if (this.totalRows === 0) return; + const target = typeof ts === 'number' ? ts : ts.getTime(); + let lo = 0; + let hi = this.totalRows - 1; + let best = 0; + let bestDiff = Infinity; + + while (lo <= hi) { + const mid = Math.floor((lo + hi) / 2); + const row = findInCache(this.cache, mid); + if (!row) break; + const rowTs = getTimestamp(row); + const diff = Math.abs(rowTs - target); + if (diff < bestDiff) { + bestDiff = diff; + best = mid; + } + if (rowTs > target) lo = mid + 1; + else hi = mid - 1; + } + this.scrollToRow(best); + } + + invalidate() { + this.lastRange = null; + this.renderRows(); + } + + clearCache() { + this.cache.clear(); + this.pending.clear(); + } + + destroy() { + this.container.removeEventListener('scroll', this.scrollHandler); + if (this.clickHandler) { + this.tbody.removeEventListener('click', this.clickHandler); + } + if (this.rafId) cancelAnimationFrame(this.rafId); + this.cache.clear(); + } +} diff --git a/js/virtual-table.test.js b/js/virtual-table.test.js new file mode 100644 index 0000000..836bf70 --- /dev/null +++ b/js/virtual-table.test.js @@ -0,0 +1,371 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { assert } from 'chai'; +import { VirtualTable } from './virtual-table.js'; + +function makeContainer(height = 280) { + const el = document.createElement('div'); + // Simulate a sized container + Object.defineProperty(el, 'clientHeight', { value: height, writable: true }); + Object.defineProperty(el, 'scrollTop', { value: 0, writable: true }); + document.body.appendChild(el); + return el; +} + +function makeColumns() { + return [ + { key: 'timestamp', label: 'Time', width: 180 }, + { key: 'status', label: 'Status', width: 60 }, + ]; +} + +function makeGetData(rows) { + const calls = []; + const fn = async (startIdx, count) => { + calls.push({ startIdx, count }); + return rows.slice(startIdx, startIdx + count); + }; + fn.calls = calls; + return fn; +} + +function renderCell(col, value) { + return value != null ? String(value) : ''; +} + +describe('VirtualTable', () => { + let container; + let vt; + + afterEach(() => { + if (vt) vt.destroy(); + if (container && container.parentNode) container.parentNode.removeChild(container); + }); + + describe('constructor', () => { + it('creates table elements in the container', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + assert.ok(container.querySelector('table')); + assert.ok(container.querySelector('thead')); + assert.ok(container.querySelector('tbody')); + }); + + it('renders column headers', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + const ths = container.querySelectorAll('thead th'); + assert.strictEqual(ths.length, 2); + assert.strictEqual(ths[0].textContent, 'Time'); + assert.strictEqual(ths[1].textContent, 'Status'); + }); + + it('sets container overflow-y to auto', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + assert.strictEqual(container.style.overflowY, 'auto'); + }); + }); + + describe('setTotalRows', () => { + it('updates spacer height', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1000); + const spacer = container.querySelector('div'); + assert.strictEqual(spacer.style.height, '28000px'); + }); + + it('uses custom row height', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, rowHeight: 40, + }); + vt.setTotalRows(500); + const spacer = container.querySelector('div'); + assert.strictEqual(spacer.style.height, '20000px'); + }); + }); + + describe('row index calculation', () => { + it('calculates start index from scroll position', () => { + container = makeContainer(280); // 10 visible rows at 28px + const getData = makeGetData(Array.from({ length: 100 }, (_, i) => ({ + timestamp: `2026-01-01 00:00:00.${String(i).padStart(3, '0')}`, + status: 200, + }))); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(100); + + // Scroll to row 20 (scrollTop = 20 * 28 = 560) + Object.defineProperty(container, 'scrollTop', { value: 560, writable: true }); + vt.invalidate(); + + const rows = container.querySelectorAll('tbody tr'); + assert.ok(rows.length > 0, 'should render some rows'); + // First row should be around index 10 (20 - overscan of 10) + const firstIdx = parseInt(rows[0].dataset.rowIdx, 10); + assert.strictEqual(firstIdx, 10, 'first rendered row should be start index minus overscan'); + }); + }); + + describe('visible range with overscan', () => { + it('renders overscan rows above and below viewport', () => { + container = makeContainer(280); + const totalRows = 200; + const allRows = Array.from({ length: totalRows }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + const getData = makeGetData(allRows); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(totalRows); + + // Scroll to middle + Object.defineProperty(container, 'scrollTop', { value: 100 * 28, writable: true }); + vt.invalidate(); + + const rows = container.querySelectorAll('tbody tr'); + // With overscan=10, visible=10, total rendered should be ~30 + assert.ok(rows.length >= 20, `should render at least 20 rows but got ${rows.length}`); + assert.ok(rows.length <= 40, `should render at most 40 rows but got ${rows.length}`); + }); + }); + + describe('cache', () => { + it('calls getData for missing rows', () => { + container = makeContainer(280); + const getData = makeGetData([]); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(50); + + // getData should have been called for the visible range + assert.ok(getData.calls.length > 0, 'should call getData'); + }); + + it('does not re-fetch cached rows', async () => { + container = makeContainer(280); + const allRows = Array.from({ length: 50 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + const getData = makeGetData(allRows); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(50); + + // Wait for initial fetch + await new Promise((r) => { + setTimeout(r, 50); + }); + const initialCalls = getData.calls.length; + + // Invalidate without scrolling — same range, data cached + vt.invalidate(); + await new Promise((r) => { + setTimeout(r, 50); + }); + assert.strictEqual(getData.calls.length, initialCalls, 'should not re-fetch cached data'); + }); + + it('clearCache empties the cache', async () => { + container = makeContainer(280); + const allRows = Array.from({ length: 50 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + const getData = makeGetData(allRows); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(50); + await new Promise((r) => { + setTimeout(r, 50); + }); + + const callsBefore = getData.calls.length; + vt.clearCache(); + vt.invalidate(); + await new Promise((r) => { + setTimeout(r, 50); + }); + assert.ok(getData.calls.length > callsBefore, 'should re-fetch after cache clear'); + }); + }); + + describe('scrollToRow', () => { + it('sets scrollTop to row index times row height', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(100); + vt.scrollToRow(25); + assert.strictEqual(container.scrollTop, 25 * 28); + }); + + it('clamps to zero for negative index', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(100); + vt.scrollToRow(-5); + assert.strictEqual(container.scrollTop, 0); + }); + }); + + describe('scrollToTimestamp', () => { + it('scrolls to the row closest to the target timestamp', async () => { + // Use 30 rows so all fit in visible + overscan range and get cached + container = makeContainer(840); // 30 rows * 28px = 840px + const rows = Array.from({ length: 30 }, (_, i) => ({ + timestamp: 1000 - i * 10, // descending: 1000, 990, 980, ... + status: 200, + })); + const getData = makeGetData(rows); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(30); + await new Promise((r) => { + setTimeout(r, 50); + }); + + vt.scrollToTimestamp(800, (row) => row.timestamp); + // Row with ts=800 is at index 20 => scrollTop = 20*28 = 560 + assert.strictEqual(container.scrollTop, 20 * 28); + }); + }); + + describe('placeholder rendering', () => { + it('adds loading-row class for rows without data', () => { + container = makeContainer(280); + // getData returns empty — simulates data not yet loaded + const getData = async () => []; + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + vt.setTotalRows(50); + + const loadingRows = container.querySelectorAll('tbody tr.loading-row'); + assert.ok(loadingRows.length > 0, 'should have loading placeholder rows'); + }); + }); + + describe('onVisibleRangeChange', () => { + it('fires with visible row indices', () => { + container = makeContainer(280); + const ranges = []; + const allRows = Array.from({ length: 50 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + vt = new VirtualTable({ + container, + columns: makeColumns(), + getData: makeGetData(allRows), + renderCell, + onVisibleRangeChange: (first, last) => ranges.push({ first, last }), + }); + vt.setTotalRows(50); + + assert.ok(ranges.length > 0, 'should fire onVisibleRangeChange'); + assert.ok(ranges[0].first >= 0); + assert.ok(ranges[0].last > ranges[0].first); + }); + }); + + describe('onRowClick', () => { + it('fires when a row is clicked', async () => { + container = makeContainer(280); + const allRows = Array.from({ length: 50 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + const clicks = []; + vt = new VirtualTable({ + container, + columns: makeColumns(), + getData: makeGetData(allRows), + renderCell, + onRowClick: (idx, row) => clicks.push({ idx, row }), + }); + vt.setTotalRows(50); + await new Promise((r) => { + setTimeout(r, 50); + }); + + // Simulate click on first rendered row's td + const td = container.querySelector('tbody tr[data-row-idx] td'); + if (td) td.click(); + assert.ok(clicks.length > 0, 'should fire onRowClick'); + }); + }); + + describe('setColumns', () => { + it('re-renders header with new columns', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setColumns([ + { key: 'a', label: 'A' }, + { key: 'b', label: 'B' }, + { key: 'c', label: 'C' }, + ]); + const ths = container.querySelectorAll('thead th'); + assert.strictEqual(ths.length, 3); + assert.strictEqual(ths[2].textContent, 'C'); + }); + }); + + describe('pinned columns', () => { + it('applies sticky positioning to pinned header cells', () => { + container = makeContainer(); + const cols = [ + { + key: 'ts', label: 'Time', pinned: true, width: 180, + }, + { key: 'status', label: 'Status', width: 60 }, + ]; + vt = new VirtualTable({ + container, columns: cols, getData: makeGetData([]), renderCell, + }); + const th = container.querySelector('thead th'); + assert.strictEqual(th.style.position, 'sticky'); + assert.strictEqual(th.style.left, '0px'); + }); + }); + + describe('destroy', () => { + it('cleans up without errors', () => { + container = makeContainer(); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.destroy(); + // Should not throw + vt = null; + }); + }); +}); diff --git a/sql/queries/logs-at.sql b/sql/queries/logs-at.sql new file mode 100644 index 0000000..24a7144 --- /dev/null +++ b/sql/queries/logs-at.sql @@ -0,0 +1,5 @@ +SELECT {{columns}} +FROM {{database}}.{{table}} +WHERE {{timeFilter}} AND timestamp <= toDateTime64('{{cursor}}', 3) {{hostFilter}} {{facetFilters}} {{additionalWhereClause}} +ORDER BY timestamp DESC +LIMIT {{pageSize}} From 873725b705165b08c43dcbc632377d208b42c4d5 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 12 Feb 2026 18:05:37 +0100 Subject: [PATCH 04/28] fix: extract chart drawing helpers and improve test coverage Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/chart-draw.js | 176 +++++++++++++++++++++++++++++++++++++ js/chart.js | 160 +-------------------------------- js/request-context.test.js | 15 ++++ 3 files changed, 194 insertions(+), 157 deletions(-) create mode 100644 js/chart-draw.js diff --git a/js/chart-draw.js b/js/chart-draw.js new file mode 100644 index 0000000..df12318 --- /dev/null +++ b/js/chart-draw.js @@ -0,0 +1,176 @@ +/* + * Copyright 2025 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/** + * Canvas drawing helpers for the chart module. + * Extracted from chart.js to stay within the max-lines lint limit. + */ + +import { formatNumber } from './format.js'; +import { addAnomalyBounds, parseUTC } from './chart-state.js'; + +/** + * Draw Y axis with grid lines and labels + */ +export function drawYAxis(ctx, chartDimensions, cssVar, minValue, maxValue) { + const { + width, height, padding, chartHeight, labelInset, + } = chartDimensions; + ctx.fillStyle = cssVar('--text-secondary'); + ctx.font = '11px -apple-system, sans-serif'; + ctx.textAlign = 'left'; + + for (let i = 1; i <= 4; i += 1) { + const val = minValue + (maxValue - minValue) * (i / 4); + const y = height - padding.bottom - ((chartHeight * i) / 4); + + ctx.strokeStyle = cssVar('--grid-line'); + ctx.beginPath(); + ctx.moveTo(padding.left, y); + ctx.lineTo(width - padding.right, y); + ctx.stroke(); + + ctx.fillStyle = cssVar('--text-secondary'); + ctx.fillText(formatNumber(val), padding.left + labelInset, y - 4); + } +} + +/** + * Draw X axis labels + */ +// eslint-disable-next-line max-len +export function drawXAxisLabels(ctx, data, chartDimensions, intendedStartTime, intendedTimeRange, cssVar) { + const { + width, height, padding, chartWidth, labelInset, + } = chartDimensions; + ctx.fillStyle = cssVar('--text-secondary'); + const isMobile = width < 500; + const tickIndices = isMobile + ? [0, Math.floor((data.length - 1) / 2), data.length - 1] + : Array.from({ length: 6 }, (_, idx) => Math.round((idx * (data.length - 1)) / 5)); + + const validIndices = tickIndices.filter((i) => i < data.length); + for (const i of validIndices) { + const time = parseUTC(data[i].t); + const elapsed = time.getTime() - intendedStartTime; + const x = padding.left + (elapsed / intendedTimeRange) * chartWidth; + const timeStr = time.toLocaleTimeString('en-US', { + hour: '2-digit', minute: '2-digit', hour12: false, timeZone: 'UTC', + }); + const showDate = intendedTimeRange > 24 * 60 * 60 * 1000; + const label = showDate + ? `${time.toLocaleDateString('en-US', { month: 'short', day: 'numeric', timeZone: 'UTC' })}, ${timeStr}` + : timeStr; + const yPos = height - padding.bottom + 20; + + if (i === 0) { + ctx.textAlign = 'left'; + ctx.fillText(label, padding.left + labelInset, yPos); + } else if (i === data.length - 1) { + ctx.textAlign = 'right'; + ctx.fillText(`${label} (UTC)`, width - padding.right - labelInset, yPos); + } else { + ctx.textAlign = 'center'; + ctx.fillText(label, x, yPos); + } + } +} + +/** + * Draw anomaly highlight for a detected step + */ +export function drawAnomalyHighlight(ctx, step, data, chartDimensions, getX, getY, stacks) { + const { height, padding, chartWidth } = chartDimensions; + const { stackedServer, stackedClient, stackedOk } = stacks; + + const startX = getX(step.startIndex); + const endX = getX(step.endIndex); + const minBandWidth = Math.max((chartWidth / data.length) * 2, 16); + const bandPadding = minBandWidth / 2; + const bandLeft = startX - bandPadding; + const bandRight = step.startIndex === step.endIndex ? startX + bandPadding : endX + bandPadding; + + const startTime = parseUTC(data[step.startIndex].t); + const endTime = parseUTC(data[step.endIndex].t); + addAnomalyBounds({ + left: bandLeft, right: bandRight, startTime, endTime, rank: step.rank, + }); + + const opacityMultiplier = step.rank === 1 ? 1 : 0.7; + const categoryColors = { red: [240, 68, 56], yellow: [247, 144, 9], green: [18, 183, 106] }; + const [cr, cg, cb] = categoryColors[step.category] || categoryColors.green; + + const seriesBounds = { + red: [(i) => getY(stackedServer[i]), () => getY(0)], + yellow: [(i) => getY(stackedClient[i]), (i) => getY(stackedServer[i])], + green: [(i) => getY(stackedOk[i]), (i) => getY(stackedClient[i])], + }; + const [getSeriesTop, getSeriesBottom] = seriesBounds[step.category] || seriesBounds.green; + + const points = []; + for (let i = step.startIndex; i <= step.endIndex; i += 1) { + points.push({ x: getX(i), y: getSeriesTop(i) }); + } + for (let i = step.endIndex; i >= step.startIndex; i -= 1) { + points.push({ x: getX(i), y: getSeriesBottom(i) }); + } + + ctx.fillStyle = `rgba(${cr}, ${cg}, ${cb}, ${0.35 * opacityMultiplier})`; + ctx.beginPath(); + ctx.moveTo(points[0].x, points[0].y); + for (let i = 1; i < points.length; i += 1) ctx.lineTo(points[i].x, points[i].y); + ctx.closePath(); + ctx.fill(); + + ctx.strokeStyle = `rgba(${cr}, ${cg}, ${cb}, 0.8)`; + ctx.lineWidth = 1.5; + ctx.setLineDash([4, 4]); + [bandLeft, bandRight].forEach((bx) => { + ctx.beginPath(); + ctx.moveTo(bx, padding.top); + ctx.lineTo(bx, height - padding.bottom); + ctx.stroke(); + }); + ctx.setLineDash([]); + + const mag = step.magnitude; + const magnitudeLabel = mag >= 1 + ? `${mag >= 10 ? Math.round(mag) : mag.toFixed(1).replace(/\.0$/, '')}x` + : `${Math.round(mag * 100)}%`; + ctx.font = '500 11px -apple-system, sans-serif'; + ctx.textAlign = 'center'; + ctx.fillStyle = `rgb(${cr}, ${cg}, ${cb})`; + const arrow = step.type === 'spike' ? '\u25B2' : '\u25BC'; + ctx.fillText(`${step.rank} ${arrow} ${magnitudeLabel}`, (bandLeft + bandRight) / 2, padding.top + 12); +} + +/** + * Draw a stacked area with line on top + */ +export function drawStackedArea(ctx, data, getX, getY, topStack, bottomStack, colors) { + if (!topStack.some((v, i) => v > bottomStack[i])) return; + + ctx.beginPath(); + ctx.moveTo(getX(0), getY(bottomStack[0])); + for (let i = 0; i < data.length; i += 1) ctx.lineTo(getX(i), getY(topStack[i])); + for (let i = data.length - 1; i >= 0; i -= 1) ctx.lineTo(getX(i), getY(bottomStack[i])); + ctx.closePath(); + ctx.fillStyle = colors.fill; + ctx.fill(); + + ctx.beginPath(); + ctx.moveTo(getX(0), getY(topStack[0])); + for (let i = 1; i < data.length; i += 1) ctx.lineTo(getX(i), getY(topStack[i])); + ctx.strokeStyle = colors.line; + ctx.lineWidth = 2; + ctx.stroke(); +} diff --git a/js/chart.js b/js/chart.js index 2736178..7e7ce88 100644 --- a/js/chart.js +++ b/js/chart.js @@ -42,6 +42,9 @@ import { getReleasesInRange, renderReleaseShips, getShipAtPoint, showReleaseTooltip, hideReleaseTooltip, } from './releases.js'; import { investigateTimeRange, clearSelectionHighlights } from './anomaly-investigation.js'; +import { + drawYAxis, drawXAxisLabels, drawAnomalyHighlight, drawStackedArea, +} from './chart-draw.js'; import { setNavigationCallback, getNavigationCallback, @@ -51,7 +54,6 @@ import { setLastChartData, getLastChartData, getDataAtTime, - addAnomalyBounds, resetAnomalyBounds, setDetectedSteps, getDetectedSteps, @@ -132,162 +134,6 @@ function initChartCanvas() { return { canvas, ctx, rect }; } -/** - * Draw Y axis with grid lines and labels - */ -function drawYAxis(ctx, chartDimensions, cssVar, minValue, maxValue) { - const { - width, height, padding, chartHeight, labelInset, - } = chartDimensions; - ctx.fillStyle = cssVar('--text-secondary'); - ctx.font = '11px -apple-system, sans-serif'; - ctx.textAlign = 'left'; - - for (let i = 1; i <= 4; i += 1) { - const val = minValue + (maxValue - minValue) * (i / 4); - const y = height - padding.bottom - ((chartHeight * i) / 4); - - ctx.strokeStyle = cssVar('--grid-line'); - ctx.beginPath(); - ctx.moveTo(padding.left, y); - ctx.lineTo(width - padding.right, y); - ctx.stroke(); - - ctx.fillStyle = cssVar('--text-secondary'); - ctx.fillText(formatNumber(val), padding.left + labelInset, y - 4); - } -} - -/** - * Draw X axis labels - */ -function drawXAxisLabels(ctx, data, chartDimensions, intendedStartTime, intendedTimeRange, cssVar) { - const { - width, height, padding, chartWidth, labelInset, - } = chartDimensions; - ctx.fillStyle = cssVar('--text-secondary'); - const isMobile = width < 500; - const tickIndices = isMobile - ? [0, Math.floor((data.length - 1) / 2), data.length - 1] - : Array.from({ length: 6 }, (_, idx) => Math.round((idx * (data.length - 1)) / 5)); - - const validIndices = tickIndices.filter((i) => i < data.length); - for (const i of validIndices) { - const time = parseUTC(data[i].t); - const elapsed = time.getTime() - intendedStartTime; - const x = padding.left + (elapsed / intendedTimeRange) * chartWidth; - const timeStr = time.toLocaleTimeString('en-US', { - hour: '2-digit', minute: '2-digit', hour12: false, timeZone: 'UTC', - }); - const showDate = intendedTimeRange > 24 * 60 * 60 * 1000; - const label = showDate - ? `${time.toLocaleDateString('en-US', { month: 'short', day: 'numeric', timeZone: 'UTC' })}, ${timeStr}` - : timeStr; - const yPos = height - padding.bottom + 20; - - if (i === 0) { - ctx.textAlign = 'left'; - ctx.fillText(label, padding.left + labelInset, yPos); - } else if (i === data.length - 1) { - ctx.textAlign = 'right'; - ctx.fillText(`${label} (UTC)`, width - padding.right - labelInset, yPos); - } else { - ctx.textAlign = 'center'; - ctx.fillText(label, x, yPos); - } - } -} - -/** - * Draw anomaly highlight for a detected step - */ -function drawAnomalyHighlight(ctx, step, data, chartDimensions, getX, getY, stacks) { - const { height, padding, chartWidth } = chartDimensions; - const { stackedServer, stackedClient, stackedOk } = stacks; - - const startX = getX(step.startIndex); - const endX = getX(step.endIndex); - const minBandWidth = Math.max((chartWidth / data.length) * 2, 16); - const bandPadding = minBandWidth / 2; - const bandLeft = startX - bandPadding; - const bandRight = step.startIndex === step.endIndex ? startX + bandPadding : endX + bandPadding; - - const startTime = parseUTC(data[step.startIndex].t); - const endTime = parseUTC(data[step.endIndex].t); - addAnomalyBounds({ - left: bandLeft, right: bandRight, startTime, endTime, rank: step.rank, - }); - - const opacityMultiplier = step.rank === 1 ? 1 : 0.7; - const categoryColors = { red: [240, 68, 56], yellow: [247, 144, 9], green: [18, 183, 106] }; - const [cr, cg, cb] = categoryColors[step.category] || categoryColors.green; - - const seriesBounds = { - red: [(i) => getY(stackedServer[i]), () => getY(0)], - yellow: [(i) => getY(stackedClient[i]), (i) => getY(stackedServer[i])], - green: [(i) => getY(stackedOk[i]), (i) => getY(stackedClient[i])], - }; - const [getSeriesTop, getSeriesBottom] = seriesBounds[step.category] || seriesBounds.green; - - const points = []; - for (let i = step.startIndex; i <= step.endIndex; i += 1) { - points.push({ x: getX(i), y: getSeriesTop(i) }); - } - for (let i = step.endIndex; i >= step.startIndex; i -= 1) { - points.push({ x: getX(i), y: getSeriesBottom(i) }); - } - - ctx.fillStyle = `rgba(${cr}, ${cg}, ${cb}, ${0.35 * opacityMultiplier})`; - ctx.beginPath(); - ctx.moveTo(points[0].x, points[0].y); - for (let i = 1; i < points.length; i += 1) ctx.lineTo(points[i].x, points[i].y); - ctx.closePath(); - ctx.fill(); - - ctx.strokeStyle = `rgba(${cr}, ${cg}, ${cb}, 0.8)`; - ctx.lineWidth = 1.5; - ctx.setLineDash([4, 4]); - [bandLeft, bandRight].forEach((bx) => { - ctx.beginPath(); - ctx.moveTo(bx, padding.top); - ctx.lineTo(bx, height - padding.bottom); - ctx.stroke(); - }); - ctx.setLineDash([]); - - const mag = step.magnitude; - const magnitudeLabel = mag >= 1 - ? `${mag >= 10 ? Math.round(mag) : mag.toFixed(1).replace(/\.0$/, '')}x` - : `${Math.round(mag * 100)}%`; - ctx.font = '500 11px -apple-system, sans-serif'; - ctx.textAlign = 'center'; - ctx.fillStyle = `rgb(${cr}, ${cg}, ${cb})`; - const arrow = step.type === 'spike' ? '\u25B2' : '\u25BC'; - ctx.fillText(`${step.rank} ${arrow} ${magnitudeLabel}`, (bandLeft + bandRight) / 2, padding.top + 12); -} - -/** - * Draw a stacked area with line on top - */ -function drawStackedArea(ctx, data, getX, getY, topStack, bottomStack, colors) { - if (!topStack.some((v, i) => v > bottomStack[i])) return; - - ctx.beginPath(); - ctx.moveTo(getX(0), getY(bottomStack[0])); - for (let i = 0; i < data.length; i += 1) ctx.lineTo(getX(i), getY(topStack[i])); - for (let i = data.length - 1; i >= 0; i -= 1) ctx.lineTo(getX(i), getY(bottomStack[i])); - ctx.closePath(); - ctx.fillStyle = colors.fill; - ctx.fill(); - - ctx.beginPath(); - ctx.moveTo(getX(0), getY(topStack[0])); - for (let i = 1; i < data.length; i += 1) ctx.lineTo(getX(i), getY(topStack[i])); - ctx.strokeStyle = colors.line; - ctx.lineWidth = 2; - ctx.stroke(); -} - export function renderChart(data) { setLastChartData(data); resetAnomalyBounds(); diff --git a/js/request-context.test.js b/js/request-context.test.js index dae6af0..71bf6b8 100644 --- a/js/request-context.test.js +++ b/js/request-context.test.js @@ -53,6 +53,21 @@ describe('request-context', () => { } }); + it('returns already-aborted signal in fallback when input is pre-aborted', () => { + const originalAny = AbortSignal.any; + AbortSignal.any = undefined; + try { + const controllerA = new AbortController(); + controllerA.abort(); + const controllerB = new AbortController(); + const merged = mergeAbortSignals([controllerA.signal, controllerB.signal]); + + assert.isTrue(merged.aborted); + } finally { + AbortSignal.any = originalAny; + } + }); + it('returns undefined for empty signal list', () => { const merged = mergeAbortSignals([]); assert.isUndefined(merged); From 9e49c7634679eab7ccb4fefe5b80ee78a324d320 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 12 Feb 2026 19:59:52 +0100 Subject: [PATCH 05/28] fix: virtual table layout, scroll direction, and column alignment - Replace absolute-positioned rows with padding-based virtual scrolling - Add table-layout:fixed with colgroup for column alignment - Add seedCache to avoid double-fetch on initial load - Fix scroll direction and column overflow handling Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 5 ++- js/logs.js | 3 ++ js/virtual-table.js | 44 ++++++++++--------- js/virtual-table.test.js | 92 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 117 insertions(+), 27 deletions(-) diff --git a/css/logs.css b/css/logs.css index 80286cd..d21f696 100644 --- a/css/logs.css +++ b/css/logs.css @@ -48,6 +48,7 @@ .logs-table { width: 100%; border-collapse: collapse; + table-layout: fixed; font-size: 12px; font-family: 'SF Mono', 'Monaco', 'Inconsolata', 'Roboto Mono', monospace; } @@ -61,6 +62,8 @@ position: sticky; top: 0; white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; cursor: pointer; user-select: none; } @@ -105,7 +108,7 @@ padding: 8px 12px; border-bottom: 1px solid var(--border); vertical-align: top; - max-width: 300px; + max-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; diff --git a/js/logs.js b/js/logs.js index 0934f0d..b36ab86 100644 --- a/js/logs.js +++ b/js/logs.js @@ -653,6 +653,9 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) if (virtualTable) { virtualTable.setColumns(buildVirtualColumns(currentColumns)); + // Seed VirtualTable cache with pre-fetched page 0 to avoid re-fetch + virtualTable.seedCache(0, rows); + // Estimate total rows: use chart data if available, otherwise extrapolate const estimated = estimateTotalRows(); const total = estimated > rows.length ? estimated : rows.length * 10; diff --git a/js/virtual-table.js b/js/virtual-table.js index 81bf029..9358341 100644 --- a/js/virtual-table.js +++ b/js/virtual-table.js @@ -81,17 +81,12 @@ export class VirtualTable { initDom() { this.container.style.overflowY = 'auto'; - this.container.style.position = 'relative'; - - this.spacer = document.createElement('div'); - this.spacer.style.width = '1px'; - this.spacer.style.height = '0px'; - this.spacer.style.pointerEvents = 'none'; this.table = document.createElement('table'); this.table.className = 'logs-table'; - this.table.style.position = 'sticky'; - this.table.style.top = '0'; + + this.colgroup = document.createElement('colgroup'); + this.table.appendChild(this.colgroup); this.thead = document.createElement('thead'); this.tbody = document.createElement('tbody'); @@ -99,19 +94,25 @@ export class VirtualTable { this.table.appendChild(this.tbody); this.container.appendChild(this.table); - this.container.appendChild(this.spacer); this.updateHeader(); } updateHeader() { + // Build colgroup for deterministic column widths + this.colgroup.innerHTML = ''; + for (const col of this.columns) { + const colEl = document.createElement('col'); + if (col.width) colEl.style.width = `${col.width}px`; + this.colgroup.appendChild(colEl); + } + const tr = document.createElement('tr'); let pinnedLeft = 0; for (const col of this.columns) { const th = document.createElement('th'); th.textContent = col.label || col.key; th.title = col.key; - if (col.width) th.style.width = `${col.width}px`; if (col.pinned) { th.style.position = 'sticky'; th.style.left = `${pinnedLeft}px`; @@ -167,6 +168,8 @@ export class VirtualTable { renderRows() { if (this.totalRows === 0) { + this.tbody.style.paddingTop = '0px'; + this.tbody.style.paddingBottom = '0px'; this.tbody.innerHTML = ''; this.lastRange = null; return; @@ -187,22 +190,16 @@ export class VirtualTable { for (let i = start; i < end; i += 1) { const row = findInCache(this.cache, i); - const top = i * this.rowHeight; if (row) { - html += ``; + html += ``; for (const col of this.columns) { const sty = makeCellStyle(col, offsets); html += `${this.renderCellFn(col, row[col.key], row)}`; } html += ''; } else { - html += ``; + html += ``; for (const col of this.columns) { html += ` `; } @@ -211,8 +208,9 @@ export class VirtualTable { } } - this.tbody.style.position = 'relative'; - this.tbody.style.height = `${this.totalRows * this.rowHeight}px`; + // Padding-based virtual scroll: push visible rows into correct position + this.tbody.style.paddingTop = `${start * this.rowHeight}px`; + this.tbody.style.paddingBottom = `${Math.max(0, (this.totalRows - end) * this.rowHeight)}px`; this.tbody.innerHTML = html; if (fetchStart !== -1) { @@ -261,7 +259,6 @@ export class VirtualTable { setTotalRows(n) { this.totalRows = n; - this.spacer.style.height = `${n * this.rowHeight}px`; this.lastRange = null; this.renderRows(); } @@ -306,6 +303,10 @@ export class VirtualTable { this.renderRows(); } + seedCache(startIdx, rows) { + this.cache.set(startIdx, { startIdx, rows }); + } + clearCache() { this.cache.clear(); this.pending.clear(); @@ -318,5 +319,6 @@ export class VirtualTable { } if (this.rafId) cancelAnimationFrame(this.rafId); this.cache.clear(); + this.container.innerHTML = ''; } } diff --git a/js/virtual-table.test.js b/js/virtual-table.test.js index 836bf70..5773e76 100644 --- a/js/virtual-table.test.js +++ b/js/virtual-table.test.js @@ -83,14 +83,18 @@ describe('VirtualTable', () => { }); describe('setTotalRows', () => { - it('updates spacer height', () => { + it('sets tbody padding to create virtual scroll space', () => { container = makeContainer(); vt = new VirtualTable({ container, columns: makeColumns(), getData: makeGetData([]), renderCell, }); vt.setTotalRows(1000); - const spacer = container.querySelector('div'); - assert.strictEqual(spacer.style.height, '28000px'); + const { tbody } = vt; + const topPad = parseInt(tbody.style.paddingTop, 10); + const bottomPad = parseInt(tbody.style.paddingBottom, 10); + const renderedRows = tbody.querySelectorAll('tr').length; + // Total virtual height = topPad + renderedRows*rowHeight + bottomPad + assert.strictEqual(topPad + renderedRows * 28 + bottomPad, 1000 * 28); }); it('uses custom row height', () => { @@ -99,8 +103,11 @@ describe('VirtualTable', () => { container, columns: makeColumns(), getData: makeGetData([]), renderCell, rowHeight: 40, }); vt.setTotalRows(500); - const spacer = container.querySelector('div'); - assert.strictEqual(spacer.style.height, '20000px'); + const { tbody } = vt; + const topPad = parseInt(tbody.style.paddingTop, 10); + const bottomPad = parseInt(tbody.style.paddingBottom, 10); + const renderedRows = tbody.querySelectorAll('tr').length; + assert.strictEqual(topPad + renderedRows * 40 + bottomPad, 500 * 40); }); }); @@ -212,6 +219,81 @@ describe('VirtualTable', () => { }); assert.ok(getData.calls.length > callsBefore, 'should re-fetch after cache clear'); }); + + it('seedCache pre-populates rows so getData is not called', () => { + container = makeContainer(280); + const allRows = Array.from({ length: 20 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + const getData = makeGetData([]); + vt = new VirtualTable({ + container, columns: makeColumns(), getData, renderCell, + }); + + // Seed cache before setting total rows + vt.seedCache(0, allRows); + const callsBefore = getData.calls.length; + vt.setTotalRows(20); + + // Should render cached rows without calling getData + const rows = container.querySelectorAll('tbody tr:not(.loading-row)'); + assert.ok(rows.length > 0, 'should render seeded rows'); + assert.strictEqual(getData.calls.length, callsBefore, 'should not call getData for seeded data'); + }); + }); + + describe('layout', () => { + it('renders rows in normal flow without position absolute', async () => { + container = makeContainer(280); + const allRows = Array.from({ length: 30 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData(allRows), renderCell, + }); + vt.setTotalRows(30); + await new Promise((r) => { + setTimeout(r, 50); + }); + + const rows = container.querySelectorAll('tbody tr'); + for (const row of rows) { + assert.notInclude(row.style.position, 'absolute', 'rows should not be absolute positioned'); + } + }); + + it('uses colgroup for column widths', () => { + container = makeContainer(); + const cols = [ + { key: 'timestamp', label: 'Time', width: 180 }, + { key: 'status', label: 'Status', width: 60 }, + ]; + vt = new VirtualTable({ + container, columns: cols, getData: makeGetData([]), renderCell, + }); + const colEls = container.querySelectorAll('colgroup col'); + assert.strictEqual(colEls.length, 2); + assert.strictEqual(colEls[0].style.width, '180px'); + assert.strictEqual(colEls[1].style.width, '60px'); + }); + + it('sets tbody paddingTop and paddingBottom for virtual scroll area', () => { + container = makeContainer(280); + const allRows = Array.from({ length: 100 }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + })); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData(allRows), renderCell, + }); + vt.setTotalRows(100); + + const { tbody } = vt; + const topPad = parseInt(tbody.style.paddingTop, 10); + const bottomPad = parseInt(tbody.style.paddingBottom, 10); + // With scrollTop=0, paddingTop should be 0 (start=0) + assert.strictEqual(topPad, 0, 'paddingTop should be 0 at scroll position 0'); + assert.ok(bottomPad > 0, 'paddingBottom should be positive'); + }); }); describe('scrollToRow', () => { From 909ecc35d0b72cd5200ad1d60840dc5825b72cbf Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 12 Feb 2026 20:20:21 +0100 Subject: [PATCH 06/28] =?UTF-8?q?fix:=20virtual=20table=20UX=20=E2=80=94?= =?UTF-8?q?=20flexbox=20layout,=20column=20widths,=20scroll-to-end?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use flexbox layout for logs-active mode so the table container fills exactly the remaining viewport (no double scrollbar) - Set explicit column widths and table width for horizontal scrolling - Use timestamp interpolation for non-sequential page jumps and cap totalRows when data is exhausted - Add tests for table width calculation Agent-Id: agent-460b7b16-e489-441c-9231-4f290ebddc75 --- css/chart.css | 5 +- css/logs.css | 32 ++++++++++- js/logs.js | 112 ++++++++++++++++++++++++++++----------- js/virtual-table.js | 10 +++- js/virtual-table.test.js | 27 ++++++++++ 5 files changed, 150 insertions(+), 36 deletions(-) diff --git a/css/chart.css b/css/chart.css index d73319c..820dd98 100644 --- a/css/chart.css +++ b/css/chart.css @@ -9,10 +9,9 @@ margin: 0 -24px 24px -24px; } -/* Sticky chart when logs view is active */ +/* Chart section takes its natural size in flex layout */ .logs-active .chart-section { - position: sticky; - top: 0; + flex-shrink: 0; z-index: 20; } diff --git a/css/logs.css b/css/logs.css index d21f696..8d26548 100644 --- a/css/logs.css +++ b/css/logs.css @@ -14,6 +14,28 @@ display: block; } +/* Logs-active: make main a flex column that fills the viewport */ +.logs-active#dashboardContent { + height: 100vh; + overflow: hidden; + display: flex; + flex-direction: column; +} + +body.header-fixed .logs-active#dashboardContent { + height: calc(100vh - 70px); +} + +/* Logs View fills remaining flex space */ +.logs-active #logsView.visible { + display: flex; + flex-direction: column; + flex: 1; + min-height: 0; + overflow: hidden; + padding: 0; +} + /* Logs View */ #logsView { padding: 16px 0 24px 0; @@ -29,6 +51,14 @@ min-height: 400px; } +/* In logs-active mode, container fills remaining flex space */ +.logs-active .logs-table-container { + flex: 1; + min-height: 0; + height: auto; + overflow: auto; +} + @media (max-width: 600px) { #logsView { padding: 0; @@ -46,7 +76,7 @@ } .logs-table { - width: 100%; + min-width: 100%; border-collapse: collapse; table-layout: fixed; font-size: 12px; diff --git a/js/logs.js b/js/logs.js index b36ab86..5f29f8d 100644 --- a/js/logs.js +++ b/js/logs.js @@ -12,7 +12,9 @@ import { DATABASE } from './config.js'; import { state, setOnPinnedColumnsChange } from './state.js'; import { query, isAbortError } from './api.js'; -import { getTimeFilter, getHostFilter, getTable } from './time.js'; +import { + getTimeFilter, getHostFilter, getTable, getTimeRangeBounds, +} from './time.js'; import { getFacetFilters } from './breakdowns/index.js'; import { escapeHtml } from './utils.js'; import { formatBytes } from './format.js'; @@ -41,6 +43,18 @@ function getLogColumns(allColumns) { return [...pinned, ...preferred, ...rest]; } +const COLUMN_WIDTHS = { + timestamp: 180, + 'response.status': 70, + 'request.method': 80, + 'request.host': 200, + 'request.url': 300, + 'response.body_size': 100, + 'cdn.cache_status': 100, + 'cdn.datacenter': 100, +}; +const DEFAULT_COLUMN_WIDTH = 150; + /** * Build VirtualTable column descriptors from column name list. * @param {string[]} columns - ordered column names @@ -51,9 +65,9 @@ function buildVirtualColumns(columns) { return columns.map((col) => { const isPinned = pinned.includes(col); const label = LOG_COLUMN_SHORT_LABELS[col] || col; - const entry = { key: col, label }; + const width = COLUMN_WIDTHS[col] || DEFAULT_COLUMN_WIDTH; + const entry = { key: col, label, width }; if (isPinned) entry.pinned = true; - if (col === 'timestamp') entry.width = 180; return entry; }); } @@ -395,6 +409,58 @@ function renderCell(col, value) { return `${colorIndicator}${escaped}`; } +/** + * Find the nearest cached cursor for a given page index. + * @param {number} pageIdx + * @returns {string|null} + */ +function findCachedCursor(pageIdx) { + for (let p = pageIdx - 1; p >= 0; p -= 1) { + const prev = pageCache.get(p); + if (prev && prev.cursor) return prev.cursor; + } + return null; +} + +/** + * Interpolate a timestamp from the time range for a given row index. + * Used when no cursor chain is available for direct page jumps. + * @param {number} startIdx + * @param {number} totalRows + * @returns {string} timestamp in 'YYYY-MM-DD HH:MM:SS.mmm' format + */ +function interpolateTimestamp(startIdx, totalRows) { + const { start, end } = getTimeRangeBounds(); + const totalMs = end.getTime() - start.getTime(); + const fraction = Math.min(startIdx / totalRows, 1); + const targetTs = new Date(end.getTime() - fraction * totalMs); + const pad = (n) => String(n).padStart(2, '0'); + const ms = String(targetTs.getUTCMilliseconds()).padStart(3, '0'); + return `${targetTs.getUTCFullYear()}-${pad(targetTs.getUTCMonth() + 1)}-${pad(targetTs.getUTCDate())} ${pad(targetTs.getUTCHours())}:${pad(targetTs.getUTCMinutes())}:${pad(targetTs.getUTCSeconds())}.${ms}`; +} + +/** + * Build the SQL query for a given page, using cursor, interpolation, or initial query. + * @param {number} pageIdx + * @param {number} startIdx + * @param {Object} sqlParams + * @returns {Promise<{sql: string, isInterpolated: boolean}>} + */ +async function buildPageQuery(pageIdx, startIdx, sqlParams) { + if (pageIdx === 0) { + return { sql: await loadSql('logs', sqlParams), isInterpolated: false }; + } + + const cursor = findCachedCursor(pageIdx); + if (cursor && TIMESTAMP_RE.test(cursor)) { + return { sql: await loadSql('logs-more', { ...sqlParams, cursor }), isInterpolated: false }; + } + + const total = virtualTable ? virtualTable.totalRows : PAGE_SIZE * 10; + const interpolatedCursor = interpolateTimestamp(startIdx, total); + return { sql: await loadSql('logs-at', { ...sqlParams, cursor: interpolatedCursor }), isInterpolated: true }; +} + /** * getData callback for VirtualTable. * Fetches a page of log rows from ClickHouse using cursor-based pagination. @@ -409,42 +475,18 @@ async function getData(startIdx, count) { return page.rows.slice(offset, offset + count); } - const timeFilter = getTimeFilter(); - const hostFilter = getHostFilter(); - const facetFilters = getFacetFilters(); const sqlParams = { database: DATABASE, table: getTable(), columns: buildLogColumnsSql(state.pinnedColumns), - timeFilter, - hostFilter, - facetFilters, + timeFilter: getTimeFilter(), + hostFilter: getHostFilter(), + facetFilters: getFacetFilters(), additionalWhereClause: state.additionalWhereClause, pageSize: String(PAGE_SIZE), }; - let sql; - if (pageIdx === 0) { - // Initial page — no cursor needed - sql = await loadSql('logs', sqlParams); - } else { - // Find the cursor from the nearest previous cached page - let cursor = null; - for (let p = pageIdx - 1; p >= 0; p -= 1) { - const prev = pageCache.get(p); - if (prev && prev.cursor) { - cursor = prev.cursor; - break; - } - } - - if (cursor && TIMESTAMP_RE.test(cursor)) { - sql = await loadSql('logs-more', { ...sqlParams, cursor }); - } else { - // No cursor available — fall back to initial query - sql = await loadSql('logs', sqlParams); - } - } + const { sql, isInterpolated } = await buildPageQuery(pageIdx, startIdx, sqlParams); try { const result = await query(sql); @@ -452,6 +494,14 @@ async function getData(startIdx, count) { const cursor = rows.length > 0 ? rows[rows.length - 1].timestamp : null; pageCache.set(pageIdx, { rows, cursor }); + // Cap totalRows when a sequential (cursor-based) page is short + if (!isInterpolated && rows.length < PAGE_SIZE && virtualTable) { + const actualTotal = pageIdx * PAGE_SIZE + rows.length; + if (actualTotal < virtualTable.totalRows) { + virtualTable.setTotalRows(actualTotal); + } + } + // Update columns on first data load if (rows.length > 0 && currentColumns.length === 0) { currentColumns = getLogColumns(Object.keys(rows[0])); diff --git a/js/virtual-table.js b/js/virtual-table.js index 9358341..f3a09fc 100644 --- a/js/virtual-table.js +++ b/js/virtual-table.js @@ -101,12 +101,20 @@ export class VirtualTable { updateHeader() { // Build colgroup for deterministic column widths this.colgroup.innerHTML = ''; + let totalWidth = 0; for (const col of this.columns) { const colEl = document.createElement('col'); - if (col.width) colEl.style.width = `${col.width}px`; + const w = col.width || 120; + colEl.style.width = `${w}px`; + totalWidth += w; this.colgroup.appendChild(colEl); } + // Set explicit table width so it can exceed container (enables horizontal scroll) + if (totalWidth > 0) { + this.table.style.width = `${totalWidth}px`; + } + const tr = document.createElement('tr'); let pinnedLeft = 0; for (const col of this.columns) { diff --git a/js/virtual-table.test.js b/js/virtual-table.test.js index 5773e76..43020e6 100644 --- a/js/virtual-table.test.js +++ b/js/virtual-table.test.js @@ -277,6 +277,33 @@ describe('VirtualTable', () => { assert.strictEqual(colEls[1].style.width, '60px'); }); + it('sets table width to sum of column widths for horizontal scrolling', () => { + container = makeContainer(); + const cols = [ + { key: 'timestamp', label: 'Time', width: 180 }, + { key: 'status', label: 'Status', width: 60 }, + { key: 'host', label: 'Host', width: 200 }, + ]; + vt = new VirtualTable({ + container, columns: cols, getData: makeGetData([]), renderCell, + }); + assert.strictEqual(vt.table.style.width, '440px'); + }); + + it('defaults column width to 120 when not specified', () => { + container = makeContainer(); + const cols = [ + { key: 'timestamp', label: 'Time', width: 180 }, + { key: 'other', label: 'Other' }, + ]; + vt = new VirtualTable({ + container, columns: cols, getData: makeGetData([]), renderCell, + }); + const colEls = container.querySelectorAll('colgroup col'); + assert.strictEqual(colEls[1].style.width, '120px'); + assert.strictEqual(vt.table.style.width, '300px'); + }); + it('sets tbody paddingTop and paddingBottom for virtual scroll area', () => { container = makeContainer(280); const allRows = Array.from({ length: 100 }, (_, i) => ({ From 859737f8851d2dfeb2f39bf71139105b1d88bad1 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Thu, 12 Feb 2026 20:31:26 +0100 Subject: [PATCH 07/28] =?UTF-8?q?fix:=20viewport=20overflow=20in=20logs=20?= =?UTF-8?q?view=20=E2=80=94=20use=20flex=20layout=20on=20#dashboard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the header is static (not fixed), the previous `height: 100vh` on #dashboardContent caused the page to be header-height + 100vh = overflow. Fix: make #dashboard a flex column (via :has(.logs-active)) so header + main together fill exactly 100vh. The header gets flex-shrink: 0 and main gets flex: 1 with min-height: 0, eliminating the overflow. This also removes the special body.header-fixed calc() rule since flex: 1 handles both static and fixed header cases correctly. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/css/logs.css b/css/logs.css index 8d26548..11272a1 100644 --- a/css/logs.css +++ b/css/logs.css @@ -14,16 +14,24 @@ display: block; } -/* Logs-active: make main a flex column that fills the viewport */ -.logs-active#dashboardContent { +/* Logs-active: make #dashboard a flex column so header + main fill exactly 100vh */ +#dashboard:has(.logs-active) { height: 100vh; overflow: hidden; display: flex; flex-direction: column; } -body.header-fixed .logs-active#dashboardContent { - height: calc(100vh - 70px); +#dashboard:has(.logs-active) > header { + flex-shrink: 0; +} + +.logs-active#dashboardContent { + flex: 1; + min-height: 0; + overflow: hidden; + display: flex; + flex-direction: column; } /* Logs View fills remaining flex space */ From 54315acc7805346020a95e6514bc4505e03303ca Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Thu, 12 Feb 2026 20:52:24 +0100 Subject: [PATCH 08/28] fix: logs view initialization in both URL restore and toggle paths Bug A: syncUIFromState now adds logs-active class to #dashboardContent and restores chart collapse state when showLogs is true. Bug B: toggleLogsView now re-seeds the virtual table from pageCache when data is already loaded, or triggers loadLogs() when it isn't. Agent-Id: agent-7176a9e4-6646-4585-bfd4-d29110221de7 --- js/logs.js | 73 ++++++++++++++++++++++++++------------------ js/url-state.js | 8 +++++ js/url-state.test.js | 34 +++++++++++++++++++++ 3 files changed, 86 insertions(+), 29 deletions(-) diff --git a/js/logs.js b/js/logs.js index 5f29f8d..2f22359 100644 --- a/js/logs.js +++ b/js/logs.js @@ -609,35 +609,6 @@ export function setOnShowFiltersView(callback) { onShowFiltersView = callback; } -export function toggleLogsView(saveStateToURL) { - state.showLogs = !state.showLogs; - const dashboardContent = document.getElementById('dashboardContent'); - if (state.showLogs) { - logsView.classList.add('visible'); - filtersView.classList.remove('visible'); - viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Filters'; - dashboardContent.classList.add('logs-active'); - // Restore collapse state from localStorage - const chartSection = document.querySelector('.chart-section'); - if (chartSection && localStorage.getItem('chartCollapsed') === 'true') { - chartSection.classList.add('chart-collapsed'); - updateCollapseToggleLabel(); - } - } else { - logsView.classList.remove('visible'); - filtersView.classList.add('visible'); - viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Logs'; - dashboardContent.classList.remove('logs-active'); - // Redraw chart after view becomes visible - if (onShowFiltersView) { - requestAnimationFrame(() => onShowFiltersView()); - } - // Clean up virtual table when leaving logs view - destroyVirtualTable(); - } - saveStateToURL(); -} - /** * Estimate total rows from chart data bucket counts. * @returns {number} @@ -723,3 +694,47 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) } } } + +export function toggleLogsView(saveStateToURL) { + state.showLogs = !state.showLogs; + const dashboardContent = document.getElementById('dashboardContent'); + if (state.showLogs) { + logsView.classList.add('visible'); + filtersView.classList.remove('visible'); + viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Filters'; + dashboardContent.classList.add('logs-active'); + // Restore collapse state from localStorage + const chartSection = document.querySelector('.chart-section'); + if (chartSection && localStorage.getItem('chartCollapsed') === 'true') { + chartSection.classList.add('chart-collapsed'); + updateCollapseToggleLabel(); + } + ensureVirtualTable(); + // Re-seed virtual table from page cache, or trigger a fresh load + if (state.logsReady && pageCache.size > 0) { + const page0 = pageCache.get(0); + if (page0 && page0.rows.length > 0) { + currentColumns = getLogColumns(Object.keys(page0.rows[0])); + virtualTable.setColumns(buildVirtualColumns(currentColumns)); + virtualTable.seedCache(0, page0.rows); + const estimated = estimateTotalRows(); + const total = estimated > page0.rows.length ? estimated : page0.rows.length * 10; + virtualTable.setTotalRows(total); + } + } else { + loadLogs(); + } + } else { + logsView.classList.remove('visible'); + filtersView.classList.add('visible'); + viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Logs'; + dashboardContent.classList.remove('logs-active'); + // Redraw chart after view becomes visible + if (onShowFiltersView) { + requestAnimationFrame(() => onShowFiltersView()); + } + // Clean up virtual table when leaving logs view + destroyVirtualTable(); + } + saveStateToURL(); +} diff --git a/js/url-state.js b/js/url-state.js index b2ead09..b4e0074 100644 --- a/js/url-state.js +++ b/js/url-state.js @@ -228,6 +228,14 @@ export function syncUIFromState() { elements.logsView.classList.add('visible'); elements.filtersView.classList.remove('visible'); elements.viewToggleBtn.querySelector('.menu-item-label').textContent = 'View Filters'; + // Activate flex layout for logs view + const dc = document.getElementById('dashboardContent'); + if (dc) dc.classList.add('logs-active'); + // Restore chart collapse state from localStorage + const chartSection = document.querySelector('.chart-section'); + if (chartSection && localStorage.getItem('chartCollapsed') === 'true') { + chartSection.classList.add('chart-collapsed'); + } } else { elements.logsView.classList.remove('visible'); elements.filtersView.classList.add('visible'); diff --git a/js/url-state.test.js b/js/url-state.test.js index 913bc42..406eebd 100644 --- a/js/url-state.test.js +++ b/js/url-state.test.js @@ -697,6 +697,7 @@ describe('syncUIFromState', () => { let mockElements; let titleEl; let activeFiltersEl; + let dashboardContentEl; beforeEach(() => { resetState(); @@ -746,11 +747,17 @@ describe('syncUIFromState', () => { activeFiltersEl = document.createElement('div'); activeFiltersEl.id = 'activeFilters'; document.body.appendChild(activeFiltersEl); + + // Create dashboardContent element for logs-active class + dashboardContentEl = document.createElement('div'); + dashboardContentEl.id = 'dashboardContent'; + document.body.appendChild(dashboardContentEl); }); afterEach(() => { titleEl.remove(); activeFiltersEl.remove(); + dashboardContentEl.remove(); window.history.replaceState({}, '', ORIGINAL_PATH); }); @@ -806,6 +813,33 @@ describe('syncUIFromState', () => { ); }); + it('adds logs-active class to dashboardContent when showLogs is true', () => { + state.showLogs = true; + syncUIFromState(); + assert.isTrue(dashboardContentEl.classList.contains('logs-active')); + }); + + it('does not add logs-active class when showLogs is false', () => { + state.showLogs = false; + syncUIFromState(); + assert.isFalse(dashboardContentEl.classList.contains('logs-active')); + }); + + it('restores chart collapse state from localStorage when showLogs is true', () => { + const chartSection = document.createElement('div'); + chartSection.classList.add('chart-section'); + document.body.appendChild(chartSection); + localStorage.setItem('chartCollapsed', 'true'); + try { + state.showLogs = true; + syncUIFromState(); + assert.isTrue(chartSection.classList.contains('chart-collapsed')); + } finally { + chartSection.remove(); + localStorage.removeItem('chartCollapsed'); + } + }); + it('shows filters view when showLogs is false', () => { state.showLogs = false; syncUIFromState(); From f9003aee17be35f4a5fa4a41c7588f759428f0d2 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 13:21:43 +0100 Subject: [PATCH 09/28] =?UTF-8?q?fix:=20infinite=20scroll=20=E2=80=94=20lo?= =?UTF-8?q?ad=20more=20rows=20when=20scrolling=20past=20initial=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When getData fetches a sequential page that returns exactly PAGE_SIZE rows, extend totalRows by at least one more page to allow continued scrolling. Only cap totalRows when a page returns fewer than PAGE_SIZE rows. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/logs.js | 19 ++++++-- js/virtual-table.test.js | 100 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 5 deletions(-) diff --git a/js/logs.js b/js/logs.js index 2f22359..2c4afaf 100644 --- a/js/logs.js +++ b/js/logs.js @@ -494,11 +494,20 @@ async function getData(startIdx, count) { const cursor = rows.length > 0 ? rows[rows.length - 1].timestamp : null; pageCache.set(pageIdx, { rows, cursor }); - // Cap totalRows when a sequential (cursor-based) page is short - if (!isInterpolated && rows.length < PAGE_SIZE && virtualTable) { - const actualTotal = pageIdx * PAGE_SIZE + rows.length; - if (actualTotal < virtualTable.totalRows) { - virtualTable.setTotalRows(actualTotal); + // Adjust totalRows based on how full this page is + if (!isInterpolated && virtualTable) { + if (rows.length < PAGE_SIZE) { + // Short page — cap totalRows to actual loaded count + const actualTotal = pageIdx * PAGE_SIZE + rows.length; + if (actualTotal < virtualTable.totalRows) { + virtualTable.setTotalRows(actualTotal); + } + } else { + // Full page — ensure there's scroll room for at least one more page + const minTotal = (pageIdx + 2) * PAGE_SIZE; + if (minTotal > virtualTable.totalRows) { + virtualTable.setTotalRows(minTotal); + } } } diff --git a/js/virtual-table.test.js b/js/virtual-table.test.js index 43020e6..61dac56 100644 --- a/js/virtual-table.test.js +++ b/js/virtual-table.test.js @@ -466,6 +466,106 @@ describe('VirtualTable', () => { }); }); + describe('infinite scroll (totalRows growth)', () => { + // Simulates how logs.js getData works: internally fetches a full page from + // the server, checks if the page is full/partial, and adjusts totalRows. + // The VirtualTable only sees the sliced result, but totalRows is updated + // via setTotalRows as a side effect. + const INTERNAL_PAGE_SIZE = 50; + + function makePagedGetData(opts = {}) { + const { totalAvailable = Infinity } = opts; + const cache = new Map(); + return async (startIdx, count) => { + const pageIdx = Math.floor(startIdx / INTERNAL_PAGE_SIZE); + if (!cache.has(pageIdx)) { + // Simulate server fetch of a full page + const pageStart = pageIdx * INTERNAL_PAGE_SIZE; + const available = Math.max(0, totalAvailable - pageStart); + const fetchedCount = Math.min(INTERNAL_PAGE_SIZE, available); + const fetched = Array.from({ length: fetchedCount }, (_, i) => ({ + timestamp: `row-${pageStart + i}`, status: 200, + })); + cache.set(pageIdx, fetched); + + // Adjust totalRows based on page fullness (mirrors logs.js fix) + if (fetched.length < INTERNAL_PAGE_SIZE) { + const actualTotal = pageIdx * INTERNAL_PAGE_SIZE + fetched.length; + if (actualTotal < vt.totalRows) { + vt.setTotalRows(actualTotal); + } + } else { + const minTotal = (pageIdx + 2) * INTERNAL_PAGE_SIZE; + if (minTotal > vt.totalRows) { + vt.setTotalRows(minTotal); + } + } + } + const page = cache.get(pageIdx); + const offset = startIdx - pageIdx * INTERNAL_PAGE_SIZE; + return page.slice(offset, offset + count); + }; + } + + it('grows totalRows when a full page is fetched', async () => { + container = makeContainer(280); + const initialTotal = INTERNAL_PAGE_SIZE * 2; // 100 + const fn = makePagedGetData({ totalAvailable: 500 }); + + vt = new VirtualTable({ + container, columns: makeColumns(), getData: fn, renderCell, + }); + vt.seedCache(0, Array.from({ length: INTERNAL_PAGE_SIZE }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + }))); + vt.setTotalRows(initialTotal); + + // Scroll near end of initial estimate — triggers fetch in uncached page 1 + Object.defineProperty(container, 'scrollTop', { + value: (initialTotal - 5) * 28, writable: true, + }); + vt.invalidate(); + await new Promise((r) => { + setTimeout(r, 150); + }); + + // Full page fetched for page 1 → totalRows should grow to (1+2)*50=150 + assert.ok( + vt.totalRows > initialTotal, + `totalRows should grow beyond ${initialTotal}, got ${vt.totalRows}`, + ); + }); + + it('caps totalRows when a partial page is fetched', async () => { + container = makeContainer(280); + const totalAvailable = INTERNAL_PAGE_SIZE + 10; // 60 rows total + const fn = makePagedGetData({ totalAvailable }); + + vt = new VirtualTable({ + container, columns: makeColumns(), getData: fn, renderCell, + }); + vt.seedCache(0, Array.from({ length: INTERNAL_PAGE_SIZE }, (_, i) => ({ + timestamp: `row-${i}`, status: 200, + }))); + vt.setTotalRows(500); // Large initial estimate + + // Scroll to trigger fetch for page 1 (which has only 10 rows) + Object.defineProperty(container, 'scrollTop', { + value: INTERNAL_PAGE_SIZE * 28, writable: true, + }); + vt.invalidate(); + await new Promise((r) => { + setTimeout(r, 150); + }); + + assert.strictEqual( + vt.totalRows, + totalAvailable, + `totalRows should be capped to ${totalAvailable}`, + ); + }); + }); + describe('destroy', () => { it('cleans up without errors', () => { container = makeContainer(); From 934ad1304ed2edbb67627fd7a6013b219092e60e Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Fri, 13 Feb 2026 13:39:28 +0100 Subject: [PATCH 10/28] =?UTF-8?q?fix:=20virtual=20table=20spacer=20?= =?UTF-8?q?=E2=80=94=20use=20div=20elements=20instead=20of=20tbody=20paddi?= =?UTF-8?q?ng?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CSS padding on with display:table-row-group is ignored by the browser's table layout engine, so the scroll container never grew beyond the rendered rows. Replace tbody padding with spacer
elements before and after the
inside the scroll container. Co-Authored-By: Claude Opus 4.6 --- js/virtual-table.js | 19 ++++++++++++++----- js/virtual-table.test.js | 35 ++++++++++++++++------------------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/js/virtual-table.js b/js/virtual-table.js index f3a09fc..6d6242f 100644 --- a/js/virtual-table.js +++ b/js/virtual-table.js @@ -82,6 +82,9 @@ export class VirtualTable { initDom() { this.container.style.overflowY = 'auto'; + this.spacerTop = document.createElement('div'); + this.spacerTop.className = 'vt-spacer-top'; + this.table = document.createElement('table'); this.table.className = 'logs-table'; @@ -93,7 +96,12 @@ export class VirtualTable { this.table.appendChild(this.thead); this.table.appendChild(this.tbody); + this.spacerBottom = document.createElement('div'); + this.spacerBottom.className = 'vt-spacer-bottom'; + + this.container.appendChild(this.spacerTop); this.container.appendChild(this.table); + this.container.appendChild(this.spacerBottom); this.updateHeader(); } @@ -176,8 +184,8 @@ export class VirtualTable { renderRows() { if (this.totalRows === 0) { - this.tbody.style.paddingTop = '0px'; - this.tbody.style.paddingBottom = '0px'; + this.spacerTop.style.height = '0px'; + this.spacerBottom.style.height = '0px'; this.tbody.innerHTML = ''; this.lastRange = null; return; @@ -216,9 +224,8 @@ export class VirtualTable { } } - // Padding-based virtual scroll: push visible rows into correct position - this.tbody.style.paddingTop = `${start * this.rowHeight}px`; - this.tbody.style.paddingBottom = `${Math.max(0, (this.totalRows - end) * this.rowHeight)}px`; + this.spacerTop.style.height = `${start * this.rowHeight}px`; + this.spacerBottom.style.height = `${Math.max(0, (this.totalRows - end) * this.rowHeight)}px`; this.tbody.innerHTML = html; if (fetchStart !== -1) { @@ -318,6 +325,8 @@ export class VirtualTable { clearCache() { this.cache.clear(); this.pending.clear(); + this.spacerTop.style.height = '0px'; + this.spacerBottom.style.height = '0px'; } destroy() { diff --git a/js/virtual-table.test.js b/js/virtual-table.test.js index 61dac56..e89d266 100644 --- a/js/virtual-table.test.js +++ b/js/virtual-table.test.js @@ -83,18 +83,17 @@ describe('VirtualTable', () => { }); describe('setTotalRows', () => { - it('sets tbody padding to create virtual scroll space', () => { + it('sets spacer heights to create virtual scroll space', () => { container = makeContainer(); vt = new VirtualTable({ container, columns: makeColumns(), getData: makeGetData([]), renderCell, }); vt.setTotalRows(1000); - const { tbody } = vt; - const topPad = parseInt(tbody.style.paddingTop, 10); - const bottomPad = parseInt(tbody.style.paddingBottom, 10); - const renderedRows = tbody.querySelectorAll('tr').length; - // Total virtual height = topPad + renderedRows*rowHeight + bottomPad - assert.strictEqual(topPad + renderedRows * 28 + bottomPad, 1000 * 28); + const topH = parseInt(vt.spacerTop.style.height, 10); + const bottomH = parseInt(vt.spacerBottom.style.height, 10); + const renderedRows = vt.tbody.querySelectorAll('tr').length; + // Total virtual height = topH + renderedRows*rowHeight + bottomH + assert.strictEqual(topH + renderedRows * 28 + bottomH, 1000 * 28); }); it('uses custom row height', () => { @@ -103,11 +102,10 @@ describe('VirtualTable', () => { container, columns: makeColumns(), getData: makeGetData([]), renderCell, rowHeight: 40, }); vt.setTotalRows(500); - const { tbody } = vt; - const topPad = parseInt(tbody.style.paddingTop, 10); - const bottomPad = parseInt(tbody.style.paddingBottom, 10); - const renderedRows = tbody.querySelectorAll('tr').length; - assert.strictEqual(topPad + renderedRows * 40 + bottomPad, 500 * 40); + const topH = parseInt(vt.spacerTop.style.height, 10); + const bottomH = parseInt(vt.spacerBottom.style.height, 10); + const renderedRows = vt.tbody.querySelectorAll('tr').length; + assert.strictEqual(topH + renderedRows * 40 + bottomH, 500 * 40); }); }); @@ -304,7 +302,7 @@ describe('VirtualTable', () => { assert.strictEqual(vt.table.style.width, '300px'); }); - it('sets tbody paddingTop and paddingBottom for virtual scroll area', () => { + it('sets spacer heights for virtual scroll area', () => { container = makeContainer(280); const allRows = Array.from({ length: 100 }, (_, i) => ({ timestamp: `row-${i}`, status: 200, @@ -314,12 +312,11 @@ describe('VirtualTable', () => { }); vt.setTotalRows(100); - const { tbody } = vt; - const topPad = parseInt(tbody.style.paddingTop, 10); - const bottomPad = parseInt(tbody.style.paddingBottom, 10); - // With scrollTop=0, paddingTop should be 0 (start=0) - assert.strictEqual(topPad, 0, 'paddingTop should be 0 at scroll position 0'); - assert.ok(bottomPad > 0, 'paddingBottom should be positive'); + const topH = parseInt(vt.spacerTop.style.height, 10); + const bottomH = parseInt(vt.spacerBottom.style.height, 10); + // With scrollTop=0, spacerTop should be 0 (start=0) + assert.strictEqual(topH, 0, 'spacerTop height should be 0 at scroll position 0'); + assert.ok(bottomH > 0, 'spacerBottom height should be positive'); }); }); From 20d86ed9a465f0653894fd6a1f456cc87a122c3b Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 13:49:02 +0100 Subject: [PATCH 11/28] feat: sync chart scrubber with table viewport scroll position Show a shaded range band on the chart indicating which time range is visible in the logs table. On mouseenter the band collapses to a single hover line; on mouseleave it restores the range band. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/chart.css | 7 +++++++ js/chart.js | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++- js/logs.js | 19 +++++++++++++------ 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/css/chart.css b/css/chart.css index 820dd98..67d34bc 100644 --- a/css/chart.css +++ b/css/chart.css @@ -149,6 +149,13 @@ opacity: 0.6; } +.chart-scrubber-line.range-mode { + background: rgba(59, 130, 246, 0.12); + border-left: 2px solid rgba(59, 130, 246, 0.5); + border-right: 2px solid rgba(59, 130, 246, 0.5); + opacity: 1; +} + .chart-scrubber-status { position: absolute; bottom: 0; diff --git a/js/chart.js b/js/chart.js index 7e7ce88..cc51104 100644 --- a/js/chart.js +++ b/js/chart.js @@ -92,6 +92,10 @@ let isDragging = false; let dragStartX = null; let justCompletedDrag = false; +// Scrubber range state (for table viewport sync) +let scrubberRangeStart = null; // eslint-disable-line prefer-const +let scrubberRangeEnd = null; // eslint-disable-line prefer-const + // Callback for chart→scroll sync (set by logs.js) let onChartHoverTimestamp = null; @@ -117,9 +121,47 @@ export function setScrubberPosition(timestamp) { scrubberLine.style.left = `${x}px`; scrubberLine.style.top = `${padding.top}px`; scrubberLine.style.height = `${height - padding.top - padding.bottom}px`; + scrubberLine.style.width = ''; + scrubberLine.classList.remove('range-mode'); scrubberLine.classList.add('visible'); } +/** + * Show the scrubber as a shaded band between two timestamps (table viewport sync) + * @param {Date} startTime - Start of visible range + * @param {Date} endTime - End of visible range + */ +export function setScrubberRange(startTime, endTime) { + if (!scrubberLine) return; + const startX = getXAtTime(startTime); + const endX = getXAtTime(endTime); + const chartLayout = getChartLayout(); + if (!chartLayout) return; + + const { padding, height } = chartLayout; + const { top } = padding; + const bandHeight = height - padding.top - padding.bottom; + + scrubberLine.style.left = `${Math.min(startX, endX)}px`; + scrubberLine.style.width = `${Math.max(2, Math.abs(endX - startX))}px`; + scrubberLine.style.top = `${top}px`; + scrubberLine.style.height = `${bandHeight}px`; + scrubberLine.classList.add('visible', 'range-mode'); + + // Store range so it can be restored after hover + scrubberRangeStart = startTime; + scrubberRangeEnd = endTime; +} + +/** + * Restore the scrubber range band if one was set (called on mouseleave) + */ +function restoreScrubberRange() { + if (scrubberRangeStart && scrubberRangeEnd) { + setScrubberRange(scrubberRangeStart, scrubberRangeEnd); + } +} + /** * Initialize canvas for chart rendering */ @@ -597,15 +639,23 @@ export function setupChartNavigation(callback) { // Show/hide scrubber on container hover container.addEventListener('mouseenter', () => { + // Switch from range band to single-line hover mode + scrubberLine.classList.remove('range-mode'); + scrubberLine.style.width = ''; scrubberLine.classList.add('visible'); scrubberStatusBar.classList.add('visible'); }); container.addEventListener('mouseleave', () => { - scrubberLine.classList.remove('visible'); scrubberStatusBar.classList.remove('visible'); hideReleaseTooltip(); canvas.style.cursor = ''; + // Restore range band if logs view is active + if (state.showLogs && scrubberRangeStart) { + restoreScrubberRange(); + } else { + scrubberLine.classList.remove('visible'); + } }); container.addEventListener('mousemove', (e) => { diff --git a/js/logs.js b/js/logs.js index 2c4afaf..bc90d26 100644 --- a/js/logs.js +++ b/js/logs.js @@ -24,7 +24,7 @@ import { LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, buildLogColumnsSql } from '. import { loadSql } from './sql-loader.js'; import { formatLogCell } from './templates/logs-table.js'; import { PAGE_SIZE } from './pagination.js'; -import { setScrubberPosition } from './chart.js'; +import { setScrubberPosition, setScrubberRange } from './chart.js'; import { parseUTC } from './chart-state.js'; import { VirtualTable } from './virtual-table.js'; @@ -569,11 +569,18 @@ function ensureVirtualTable() { getData, renderCell, onVisibleRangeChange(firstRow, lastRow) { - // Sync chart scrubber to the middle visible row - const midIdx = Math.floor((firstRow + lastRow) / 2); - const row = getRowFromCache(midIdx); - if (row && row.timestamp) { - setScrubberPosition(parseUTC(row.timestamp)); + // Sync chart scrubber to the visible table range + const firstRowData = getRowFromCache(firstRow); + const lastRowData = getRowFromCache(lastRow); + if (firstRowData?.timestamp && lastRowData?.timestamp) { + setScrubberRange(parseUTC(firstRowData.timestamp), parseUTC(lastRowData.timestamp)); + } else { + // Fallback to single point if we don't have both endpoints + const midIdx = Math.floor((firstRow + lastRow) / 2); + const row = getRowFromCache(midIdx); + if (row?.timestamp) { + setScrubberPosition(parseUTC(row.timestamp)); + } } }, onRowClick(idx, row) { From 796af5f073c514b70e72ab22782b813e13e37388 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 13:50:26 +0100 Subject: [PATCH 12/28] feat: restore click-to-filter on virtual table cell values Add data-action="add-filter" attributes to cells with facet mappings, enabling the existing global action handler to process filter clicks. Skip onRowClick dispatch for cells with data-action to prevent the detail modal from opening on filter clicks. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/logs.js | 19 ++++++++++++++++--- js/virtual-table.js | 2 ++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/js/logs.js b/js/logs.js index bc90d26..513c541 100644 --- a/js/logs.js +++ b/js/logs.js @@ -20,7 +20,9 @@ import { escapeHtml } from './utils.js'; import { formatBytes } from './format.js'; import { getColorForColumn } from './colors/index.js'; import { getRequestContext, isRequestCurrent } from './request-context.js'; -import { LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, buildLogColumnsSql } from './columns.js'; +import { + LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, LOG_COLUMN_TO_FACET, buildLogColumnsSql, +} from './columns.js'; import { loadSql } from './sql-loader.js'; import { formatLogCell } from './templates/logs-table.js'; import { PAGE_SIZE } from './pagination.js'; @@ -405,8 +407,19 @@ export function initChartCollapseToggle() { function renderCell(col, value) { const { displayValue, cellClass, colorIndicator } = formatLogCell(col.key, value); const escaped = escapeHtml(displayValue); - const cls = cellClass ? ` class="${cellClass}"` : ''; - return `${colorIndicator}${escaped}`; + + // Add click-to-filter attributes when column has a facet mapping and a color indicator + let actionAttrs = ''; + let extraClass = ''; + const facetMapping = LOG_COLUMN_TO_FACET[col.key]; + if (colorIndicator && facetMapping && value !== null && value !== undefined && value !== '') { + const filterValue = facetMapping.transform ? facetMapping.transform(value) : String(value); + actionAttrs = ` data-action="add-filter" data-col="${escapeHtml(facetMapping.col)}" data-value="${escapeHtml(filterValue)}" data-exclude="false"`; + extraClass = ' clickable'; + } + + const cls = cellClass || extraClass ? ` class="${(cellClass || '') + extraClass}"` : ''; + return `${colorIndicator}${escaped}`; } /** diff --git a/js/virtual-table.js b/js/virtual-table.js index 6d6242f..a14e569 100644 --- a/js/virtual-table.js +++ b/js/virtual-table.js @@ -157,6 +157,8 @@ export class VirtualTable { if (this.onRowClickFn) { this.clickHandler = (e) => { + // Let data-action clicks (e.g. add-filter) bubble to global handler + if (e.target.closest('[data-action]')) return; const tr = e.target.closest('tr[data-row-idx]'); if (!tr) return; const idx = parseInt(tr.dataset.rowIdx, 10); From 3b6a5bf7723d427ee2df99ff85d952d6e4decaee Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 13:52:06 +0100 Subject: [PATCH 13/28] fix: remove sample_hash from logs SELECT columns sample_hash is not displayed in the table and not needed for cursor-based log queries. Removing it reduces query bandwidth. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/columns.js | 2 +- js/columns.test.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/js/columns.js b/js/columns.js index ab51e3f..f446e62 100644 --- a/js/columns.js +++ b/js/columns.js @@ -186,7 +186,7 @@ export const LOG_COLUMN_SHORT_LABELS = Object.fromEntries( * Columns always included in logs queries (needed for internal use, not display). * @type {string[]} */ -const ALWAYS_NEEDED_COLUMNS = ['timestamp', 'source', 'sample_hash']; +const ALWAYS_NEEDED_COLUMNS = ['timestamp', 'source']; /** * Build the backtick-quoted column list for logs SQL queries. diff --git a/js/columns.test.js b/js/columns.test.js index 6b9655c..302eedb 100644 --- a/js/columns.test.js +++ b/js/columns.test.js @@ -24,7 +24,7 @@ describe('buildLogColumnsSql', () => { const result = buildLogColumnsSql(); assert.include(result, '`timestamp`'); assert.include(result, '`source`'); - assert.include(result, '`sample_hash`'); + assert.notInclude(result, '`sample_hash`'); }); it('includes all LOG_COLUMN_ORDER columns', () => { @@ -66,6 +66,5 @@ describe('buildLogColumnsSql', () => { const parts = result.split(', '); assert.strictEqual(parts[0], '`timestamp`'); assert.strictEqual(parts[1], '`source`'); - assert.strictEqual(parts[2], '`sample_hash`'); }); }); From 42bb4390363da32a44bb8708338b552ea85279d0 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 13:54:17 +0100 Subject: [PATCH 14/28] feat: click chart to scroll logs table to clicked timestamp Clicking on the chart area (when no anomaly or selection is present) fires a callback that scrolls the logs table to the clicked time. If logs view is not active, it toggles to logs view first, then scrolls. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/chart.js | 15 ++++++++++++++- js/dashboard-init.js | 11 ++++++++++- js/logs.js | 5 ++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/js/chart.js b/js/chart.js index cc51104..f893aa7 100644 --- a/js/chart.js +++ b/js/chart.js @@ -98,6 +98,7 @@ let scrubberRangeEnd = null; // eslint-disable-line prefer-const // Callback for chart→scroll sync (set by logs.js) let onChartHoverTimestamp = null; +let onChartClickTimestamp = null; /** * Set callback for chart hover → scroll sync @@ -107,6 +108,14 @@ export function setOnChartHoverTimestamp(callback) { onChartHoverTimestamp = callback; } +/** + * Set callback for chart click → load logs at timestamp + * @param {Function} callback - Called with Date when clicking chart + */ +export function setOnChartClickTimestamp(callback) { + onChartClickTimestamp = callback; +} + /** * Position the scrubber line at a given timestamp (called from logs.js scroll sync) * @param {Date} timestamp - Timestamp to position scrubber at @@ -774,9 +783,13 @@ export function setupChartNavigation(callback) { hideSelectionOverlay(); return; } - const anomalyBounds = getAnomalyAtX(e.clientX - canvas.getBoundingClientRect().left); + const clickX = e.clientX - canvas.getBoundingClientRect().left; + const anomalyBounds = getAnomalyAtX(clickX); if (anomalyBounds) { zoomToAnomalyByRank(anomalyBounds.rank); + } else if (onChartClickTimestamp) { + const clickTime = getTimeAtX(clickX); + if (clickTime) onChartClickTimestamp(clickTime); } return; } diff --git a/js/dashboard-init.js b/js/dashboard-init.js index febdd18..549a1b6 100644 --- a/js/dashboard-init.js +++ b/js/dashboard-init.js @@ -29,7 +29,7 @@ import { } from './timer.js'; import { loadTimeSeries, setupChartNavigation, getDetectedAnomalies, getLastChartData, - renderChart, setOnChartHoverTimestamp, + renderChart, setOnChartHoverTimestamp, setOnChartClickTimestamp, } from './chart.js'; import { loadAllBreakdowns, loadBreakdown, getBreakdowns, markSlowestFacet, resetFacetTimings, @@ -211,6 +211,15 @@ export function initDashboard(config = {}) { scrollLogsToTimestamp(timestamp); }); + // Chart click → open logs at clicked timestamp + setOnChartClickTimestamp((timestamp) => { + if (!state.showLogs) { + toggleLogsView(saveStateToURL, timestamp); + } else { + scrollLogsToTimestamp(timestamp); + } + }); + setFilterCallbacks(saveStateToURL, loadDashboard); setOnBeforeRestore(() => invalidateInvestigationCache()); setOnStateRestored(loadDashboard); diff --git a/js/logs.js b/js/logs.js index 513c541..f072715 100644 --- a/js/logs.js +++ b/js/logs.js @@ -724,7 +724,7 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) } } -export function toggleLogsView(saveStateToURL) { +export function toggleLogsView(saveStateToURL, scrollToTimestamp) { state.showLogs = !state.showLogs; const dashboardContent = document.getElementById('dashboardContent'); if (state.showLogs) { @@ -749,6 +749,9 @@ export function toggleLogsView(saveStateToURL) { const estimated = estimateTotalRows(); const total = estimated > page0.rows.length ? estimated : page0.rows.length * 10; virtualTable.setTotalRows(total); + if (scrollToTimestamp) { + scrollLogsToTimestamp(scrollToTimestamp); + } } } else { loadLogs(); From bcde0b0c8abae03433ac9640d8d27b5dce77720b Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 13:57:26 +0100 Subject: [PATCH 15/28] perf: reduce initial logs page size from 500 to 100 rows Fetch only 100 rows for the first paint, then expand to full 500-row pages on scroll. Extract cache lookup and totalRows adjustment into helper functions to reduce getData complexity. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/logs.js | 63 ++++++++++++++++++++++++++++++------------------ js/pagination.js | 1 + 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/js/logs.js b/js/logs.js index f072715..f6577f7 100644 --- a/js/logs.js +++ b/js/logs.js @@ -25,7 +25,7 @@ import { } from './columns.js'; import { loadSql } from './sql-loader.js'; import { formatLogCell } from './templates/logs-table.js'; -import { PAGE_SIZE } from './pagination.js'; +import { PAGE_SIZE, INITIAL_PAGE_SIZE } from './pagination.js'; import { setScrubberPosition, setScrubberRange } from './chart.js'; import { parseUTC } from './chart-state.js'; import { VirtualTable } from './virtual-table.js'; @@ -474,6 +474,40 @@ async function buildPageQuery(pageIdx, startIdx, sqlParams) { return { sql: await loadSql('logs-at', { ...sqlParams, cursor: interpolatedCursor }), isInterpolated: true }; } +/** + * Check the page cache for data at the given index. + * Returns the cached rows, or null if a fresh fetch is needed. + */ +function getCachedRows(pageIdx, startIdx, count) { + if (!pageCache.has(pageIdx)) return null; + const page = pageCache.get(pageIdx); + const offset = startIdx - pageIdx * PAGE_SIZE; + if (offset < page.rows.length) { + return page.rows.slice(offset, offset + count); + } + // Page 0 may be a partial initial load — allow re-fetch + // Other pages: a short cache means end of data + return pageIdx === 0 ? null : []; +} + +/** + * Adjust virtualTable totalRows after fetching a page. + */ +function adjustTotalRows(pageIdx, rowCount) { + if (!virtualTable) return; + if (rowCount < PAGE_SIZE) { + const actualTotal = pageIdx * PAGE_SIZE + rowCount; + if (actualTotal < virtualTable.totalRows) { + virtualTable.setTotalRows(actualTotal); + } + } else { + const minTotal = (pageIdx + 2) * PAGE_SIZE; + if (minTotal > virtualTable.totalRows) { + virtualTable.setTotalRows(minTotal); + } + } +} + /** * getData callback for VirtualTable. * Fetches a page of log rows from ClickHouse using cursor-based pagination. @@ -481,12 +515,8 @@ async function buildPageQuery(pageIdx, startIdx, sqlParams) { async function getData(startIdx, count) { const pageIdx = Math.floor(startIdx / PAGE_SIZE); - // Return from cache if available - if (pageCache.has(pageIdx)) { - const page = pageCache.get(pageIdx); - const offset = startIdx - pageIdx * PAGE_SIZE; - return page.rows.slice(offset, offset + count); - } + const cached = getCachedRows(pageIdx, startIdx, count); + if (cached !== null) return cached; const sqlParams = { database: DATABASE, @@ -507,21 +537,8 @@ async function getData(startIdx, count) { const cursor = rows.length > 0 ? rows[rows.length - 1].timestamp : null; pageCache.set(pageIdx, { rows, cursor }); - // Adjust totalRows based on how full this page is - if (!isInterpolated && virtualTable) { - if (rows.length < PAGE_SIZE) { - // Short page — cap totalRows to actual loaded count - const actualTotal = pageIdx * PAGE_SIZE + rows.length; - if (actualTotal < virtualTable.totalRows) { - virtualTable.setTotalRows(actualTotal); - } - } else { - // Full page — ensure there's scroll room for at least one more page - const minTotal = (pageIdx + 2) * PAGE_SIZE; - if (minTotal > virtualTable.totalRows) { - virtualTable.setTotalRows(minTotal); - } - } + if (!isInterpolated) { + adjustTotalRows(pageIdx, rows.length); } // Update columns on first data load @@ -677,7 +694,7 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) hostFilter, facetFilters, additionalWhereClause: state.additionalWhereClause, - pageSize: String(PAGE_SIZE), + pageSize: String(INITIAL_PAGE_SIZE), }); try { diff --git a/js/pagination.js b/js/pagination.js index 70d24dd..e7f60bd 100644 --- a/js/pagination.js +++ b/js/pagination.js @@ -11,6 +11,7 @@ */ export const PAGE_SIZE = 500; +export const INITIAL_PAGE_SIZE = 100; export class PaginationState { constructor(pageSize = PAGE_SIZE) { From 4e9652743935bf0e9b5e7b5190e518a2797d58d9 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 14:40:12 +0100 Subject: [PATCH 16/28] fix: bucket-aware scroll interpolation to eliminate scroll acceleration Replace the broken estimateTotalRows() (was reading nonexistent .buckets property) and linear interpolateTimestamp() with a bucket-aware approach that uses chart data for accurate scroll mapping. - Build cumulative row-count index from chart bucket counts - Binary search cumulative array for timestamp interpolation - Remove adjustTotalRows growth path that caused scroll bar jumps - Fix estimateTotalRows to parse actual chart data fields Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/logs.js | 114 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 18 deletions(-) diff --git a/js/logs.js b/js/logs.js index f6577f7..4148bb4 100644 --- a/js/logs.js +++ b/js/logs.js @@ -30,6 +30,10 @@ import { setScrubberPosition, setScrubberRange } from './chart.js'; import { parseUTC } from './chart-state.js'; import { VirtualTable } from './virtual-table.js'; +// Cached bucket index built from chart data +// eslint-disable-next-line prefer-const -- reassigned in buildBucketIndex/loadLogs +let bucketIndex = null; + const TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/; /** @@ -436,20 +440,65 @@ function findCachedCursor(pageIdx) { } /** - * Interpolate a timestamp from the time range for a given row index. - * Used when no cursor chain is available for direct page jumps. - * @param {number} startIdx + * Format a Date as 'YYYY-MM-DD HH:MM:SS.mmm' in UTC. + * @param {Date} date + * @returns {string} + */ +function formatTimestampUTC(date) { + const pad = (n) => String(n).padStart(2, '0'); + const ms = String(date.getUTCMilliseconds()).padStart(3, '0'); + return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())} ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}.${ms}`; +} + +/** + * Interpolate a timestamp for a given row index using bucket-aware lookup. + * Chart data buckets are ordered oldest→newest (ascending time), but rows are + * ordered newest→oldest (descending time, ORDER BY timestamp DESC). + * So row 0 = newest bucket (last in chart), row N = oldest bucket (first in chart). + * + * Uses binary search on the cumulative row-count array when available, + * falling back to linear interpolation across the time range. + * @param {number} startIdx - virtual row index (0 = newest) * @param {number} totalRows * @returns {string} timestamp in 'YYYY-MM-DD HH:MM:SS.mmm' format */ function interpolateTimestamp(startIdx, totalRows) { + // Bucket-aware path: binary search the cumulative array + if (bucketIndex && bucketIndex.cumulative.length > 0) { + const { cumulative } = bucketIndex; + const total = bucketIndex.totalRows; + // Clamp to valid range + const targetRow = Math.min(Math.max(startIdx, 0), total - 1); + + // cumulative is oldest→newest; cumRows is a running total in that order. + // Row 0 (newest) maps to the END of the cumulative array. + // Convert: rows from the end = targetRow → cumulative offset from end. + const rowsFromEnd = targetRow; + // rowsFromEnd=0 means the last bucket; rowsFromEnd=total-1 means the first. + // cumulativeTarget = total - rowsFromEnd = the cumRows value we seek. + const cumulativeTarget = total - rowsFromEnd; + + // Binary search: find the bucket where cumRows >= cumulativeTarget + let lo = 0; + let hi = cumulative.length - 1; + while (lo < hi) { + const mid = Math.floor((lo + hi) / 2); + if (cumulative[mid].cumRows < cumulativeTarget) { + lo = mid + 1; + } else { + hi = mid; + } + } + + return formatTimestampUTC(parseUTC(cumulative[lo].timestamp)); + } + + // Fallback: linear interpolation across the time range const { start, end } = getTimeRangeBounds(); const totalMs = end.getTime() - start.getTime(); - const fraction = Math.min(startIdx / totalRows, 1); + const fraction = Math.min(startIdx / Math.max(totalRows, 1), 1); const targetTs = new Date(end.getTime() - fraction * totalMs); - const pad = (n) => String(n).padStart(2, '0'); - const ms = String(targetTs.getUTCMilliseconds()).padStart(3, '0'); - return `${targetTs.getUTCFullYear()}-${pad(targetTs.getUTCMonth() + 1)}-${pad(targetTs.getUTCDate())} ${pad(targetTs.getUTCHours())}:${pad(targetTs.getUTCMinutes())}:${pad(targetTs.getUTCSeconds())}.${ms}`; + return formatTimestampUTC(targetTs); } /** @@ -492,6 +541,8 @@ function getCachedRows(pageIdx, startIdx, count) { /** * Adjust virtualTable totalRows after fetching a page. + * Only shrinks totalRows when a genuine short page indicates end-of-data. + * Never grows — the initial estimate from chart data is the upper bound. */ function adjustTotalRows(pageIdx, rowCount) { if (!virtualTable) return; @@ -500,12 +551,8 @@ function adjustTotalRows(pageIdx, rowCount) { if (actualTotal < virtualTable.totalRows) { virtualTable.setTotalRows(actualTotal); } - } else { - const minTotal = (pageIdx + 2) * PAGE_SIZE; - if (minTotal > virtualTable.totalRows) { - virtualTable.setTotalRows(minTotal); - } } + // Removed: growth path that caused scroll jumps by expanding totalRows } /** @@ -655,13 +702,41 @@ export function setOnShowFiltersView(callback) { onShowFiltersView = callback; } +/** + * Build a cumulative row-count index from chart data buckets. + * Each bucket has { t, cnt_ok, cnt_4xx, cnt_5xx } (strings from ClickHouse). + * Returns { cumulative: [{timestamp, cumRows, count}], totalRows } or null. + * Buckets are ordered oldest→newest (ascending time), matching chart data order. + * @param {Array} chartData + * @returns {{cumulative: Array, totalRows: number}|null} + */ +function buildBucketIndex(chartData) { + if (!chartData || chartData.length === 0) return null; + const cumulative = []; + let total = 0; + for (const bucket of chartData) { + const count = (parseInt(bucket.cnt_ok, 10) || 0) + + (parseInt(bucket.cnt_4xx, 10) || 0) + + (parseInt(bucket.cnt_5xx, 10) || 0); + total += count; + cumulative.push({ timestamp: bucket.t, cumRows: total, count }); + } + return { cumulative, totalRows: total }; +} + /** * Estimate total rows from chart data bucket counts. * @returns {number} */ function estimateTotalRows() { - if (!state.chartData || !state.chartData.buckets) return 0; - return state.chartData.buckets.reduce((sum, b) => sum + (b.total || b.count || 0), 0); + if (!state.chartData || state.chartData.length === 0) return 0; + let total = 0; + for (const b of state.chartData) { + total += (parseInt(b.cnt_ok, 10) || 0) + + (parseInt(b.cnt_4xx, 10) || 0) + + (parseInt(b.cnt_5xx, 10) || 0); + } + return total; } export async function loadLogs(requestContext = getRequestContext('dashboard')) { @@ -671,8 +746,9 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) state.logsLoading = true; state.logsReady = false; - // Reset page cache + // Reset page cache and bucket index pageCache.clear(); + bucketIndex = null; currentColumns = []; // Apply blur effect while loading @@ -723,8 +799,9 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) // Seed VirtualTable cache with pre-fetched page 0 to avoid re-fetch virtualTable.seedCache(0, rows); - // Estimate total rows: use chart data if available, otherwise extrapolate - const estimated = estimateTotalRows(); + // Build bucket index from chart data for accurate scroll mapping + bucketIndex = buildBucketIndex(state.chartData); + const estimated = bucketIndex ? bucketIndex.totalRows : estimateTotalRows(); const total = estimated > rows.length ? estimated : rows.length * 10; virtualTable.setTotalRows(total); } @@ -763,7 +840,8 @@ export function toggleLogsView(saveStateToURL, scrollToTimestamp) { currentColumns = getLogColumns(Object.keys(page0.rows[0])); virtualTable.setColumns(buildVirtualColumns(currentColumns)); virtualTable.seedCache(0, page0.rows); - const estimated = estimateTotalRows(); + bucketIndex = buildBucketIndex(state.chartData); + const estimated = bucketIndex ? bucketIndex.totalRows : estimateTotalRows(); const total = estimated > page0.rows.length ? estimated : page0.rows.length * 10; virtualTable.setTotalRows(total); if (scrollToTimestamp) { From 3e5478ac8465abd0ef4e4c6e030447492e58fdb7 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 14:55:57 +0100 Subject: [PATCH 17/28] fix: cap virtual scroll height to prevent browser-induced acceleration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When totalRows is large (500K–2M), the natural scroll height (totalRows × 28px) exceeds browser max scroll height (~33.5M px), causing scrollTop-to-row-index mapping to compress and produce scroll acceleration. Cap at 1M pixels with a scale factor so scroll position maps linearly to row indices regardless of dataset size. - Add MAX_SCROLL_HEIGHT constant (1,000,000 px) - Compute _scrollScale in setTotalRows() when natural height exceeds cap - Apply scale to computeRange(), renderRows() spacers, scrollToRow(), and evictDistantPages() - Add 6 tests for large dataset scroll behavior Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/virtual-table.js | 27 +++++++--- js/virtual-table.test.js | 107 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 6 deletions(-) diff --git a/js/virtual-table.js b/js/virtual-table.js index a14e569..ed86084 100644 --- a/js/virtual-table.js +++ b/js/virtual-table.js @@ -13,6 +13,7 @@ const DEFAULT_ROW_HEIGHT = 28; const DEFAULT_OVERSCAN = 10; const MAX_CACHED_PAGES = 20; +const MAX_SCROLL_HEIGHT = 1_000_000; function getPinnedOffsets(columns) { const offsets = new Map(); @@ -70,6 +71,7 @@ export class VirtualTable { this.onRowClickFn = onRowClick || null; this.totalRows = 0; + this._scrollScale = 1; this.cache = new Map(); this.pending = new Set(); this.rafId = null; @@ -169,14 +171,20 @@ export class VirtualTable { } } + getTotalScrollHeight() { + const natural = this.totalRows * this.rowHeight; + return natural > MAX_SCROLL_HEIGHT ? MAX_SCROLL_HEIGHT : natural; + } + computeRange() { const { scrollTop } = this.container; const viewHeight = this.container.clientHeight; + const effectiveRowHeight = this.rowHeight * this._scrollScale; const start = Math.max( 0, - Math.floor(scrollTop / this.rowHeight) - DEFAULT_OVERSCAN, + Math.floor(scrollTop / effectiveRowHeight) - DEFAULT_OVERSCAN, ); - const visible = Math.ceil(viewHeight / this.rowHeight); + const visible = Math.ceil(viewHeight / effectiveRowHeight); const end = Math.min( this.totalRows, start + visible + DEFAULT_OVERSCAN * 2, @@ -226,8 +234,9 @@ export class VirtualTable { } } - this.spacerTop.style.height = `${start * this.rowHeight}px`; - this.spacerBottom.style.height = `${Math.max(0, (this.totalRows - end) * this.rowHeight)}px`; + const effectiveRowHeight = this.rowHeight * this._scrollScale; + this.spacerTop.style.height = `${start * effectiveRowHeight}px`; + this.spacerBottom.style.height = `${Math.max(0, (this.totalRows - end) * effectiveRowHeight)}px`; this.tbody.innerHTML = html; if (fetchStart !== -1) { @@ -259,7 +268,8 @@ export class VirtualTable { evictDistantPages() { if (this.cache.size <= MAX_CACHED_PAGES) return; - const center = this.container.scrollTop / this.rowHeight; + const effectiveRowHeight = this.rowHeight * this._scrollScale; + const center = this.container.scrollTop / effectiveRowHeight; const sorted = [...this.cache.entries()] .map(([key, page]) => ({ key, @@ -276,6 +286,10 @@ export class VirtualTable { setTotalRows(n) { this.totalRows = n; + const natural = n * this.rowHeight; + this._scrollScale = natural > MAX_SCROLL_HEIGHT + ? MAX_SCROLL_HEIGHT / natural + : 1; this.lastRange = null; this.renderRows(); } @@ -288,7 +302,8 @@ export class VirtualTable { } scrollToRow(index) { - this.container.scrollTop = Math.max(0, index * this.rowHeight); + const effectiveRowHeight = this.rowHeight * this._scrollScale; + this.container.scrollTop = Math.max(0, index * effectiveRowHeight); } scrollToTimestamp(ts, getTimestamp) { diff --git a/js/virtual-table.test.js b/js/virtual-table.test.js index e89d266..f7930f9 100644 --- a/js/virtual-table.test.js +++ b/js/virtual-table.test.js @@ -563,6 +563,113 @@ describe('VirtualTable', () => { }); }); + describe('max scroll height capping', () => { + it('uses natural scroll height for small datasets (no behavioral change)', () => { + container = makeContainer(280); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1000); + // 1000 * 28 = 28000, well under MAX_SCROLL_HEIGHT (1_000_000) + assert.strictEqual(vt.getTotalScrollHeight(), 1000 * 28); + // Spacer heights should use original row height + const topH = parseInt(vt.spacerTop.style.height, 10); + const bottomH = parseInt(vt.spacerBottom.style.height, 10); + const renderedRows = vt.tbody.querySelectorAll('tr').length; + assert.strictEqual(topH + renderedRows * 28 + bottomH, 1000 * 28); + }); + + it('caps total scroll height for large datasets', () => { + container = makeContainer(280); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1_000_000); + // 1_000_000 * 28 = 28_000_000, exceeds MAX_SCROLL_HEIGHT (1_000_000) + assert.strictEqual(vt.getTotalScrollHeight(), 1_000_000); + }); + + it('computeRange returns correct indices for large datasets', () => { + container = makeContainer(280); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1_000_000); + + // Scroll to 50% of the total scroll height + const totalScrollHeight = vt.getTotalScrollHeight(); + Object.defineProperty(container, 'scrollTop', { + value: totalScrollHeight / 2, writable: true, + }); + const range = vt.computeRange(); + // Start (before overscan subtraction) should be near 500_000 + const rawStart = range.start + 10; + assert.closeTo( + rawStart, + 500_000, + 1000, + `mid-scroll should map to ~row 500000, got ${rawStart}`, + ); + }); + + it('scrollToRow maps large indices correctly', () => { + container = makeContainer(280); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1_000_000); + + vt.scrollToRow(500_000); + const totalScrollHeight = vt.getTotalScrollHeight(); + const expected = 500_000 * (totalScrollHeight / 1_000_000); + assert.closeTo( + container.scrollTop, + expected, + 1, + 'scrollToRow(500000) should set scrollTop to ~50% of total scroll height', + ); + }); + + it('round-trips scrollToRow then computeRange for large indices', () => { + container = makeContainer(280); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1_000_000); + + const targetRow = 750_000; + vt.scrollToRow(targetRow); + const range = vt.computeRange(); + const rawStart = range.start + 10; + // Should be within a few rows of the target + assert.closeTo( + rawStart, + targetRow, + 2, + `round-trip should recover row ${targetRow}, got ${rawStart}`, + ); + }); + + it('total spacer height is capped for large datasets', () => { + container = makeContainer(280); + vt = new VirtualTable({ + container, columns: makeColumns(), getData: makeGetData([]), renderCell, + }); + vt.setTotalRows(1_000_000); + const topH = parseFloat(vt.spacerTop.style.height); + const bottomH = parseFloat(vt.spacerBottom.style.height); + // spacerTop + rendered area + spacerBottom should be within total scroll height + assert.ok( + topH + bottomH <= vt.getTotalScrollHeight(), + 'spacer heights should not exceed getTotalScrollHeight()', + ); + assert.ok( + vt.getTotalScrollHeight() <= 1_000_000, + 'total scroll height should be capped at 1_000_000', + ); + }); + }); + describe('destroy', () => { it('cleans up without errors', () => { container = makeContainer(); From 187d0bc01fe1894c1f5cfb3168f3a305ac549fde Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 15:31:20 +0100 Subject: [PATCH 18/28] feat: bucket-row table replaces virtual scroll for smooth native scrolling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the spacer-based VirtualTable approach (which caused scroll acceleration due to browser max scroll height limits) with a simpler architecture: one per chart bucket (~200-1000 rows). The browser handles this natively with zero scroll issues. - Each bucket-row height proportional to its row count (count × 28px) - Bucket rows have timestamp IDs for scrollIntoView navigation - Scrubber sync via passive scroll listener on bucket visibility - Proportional scaling when total height exceeds 10M pixels - Phase 1: placeholder content only ("342 rows" per bucket) Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 20 ++++ js/logs.js | 263 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 216 insertions(+), 67 deletions(-) diff --git a/css/logs.css b/css/logs.css index 11272a1..c4df646 100644 --- a/css/logs.css +++ b/css/logs.css @@ -224,6 +224,26 @@ color: transparent; } +/* Bucket-row table placeholder rows */ +.bucket-table .bucket-row { + cursor: default; +} + +.bucket-table .bucket-row .bucket-placeholder { + max-width: none; + vertical-align: middle; + text-align: center; + color: var(--text-secondary); + font-size: 11px; + border-bottom: 1px solid var(--border); + padding: 0 12px; + white-space: nowrap; +} + +.bucket-table .bucket-row:hover .bucket-placeholder { + background: var(--bg); +} + /* Copy feedback toast */ .copy-feedback { position: fixed; diff --git a/js/logs.js b/js/logs.js index 4148bb4..57f3045 100644 --- a/js/logs.js +++ b/js/logs.js @@ -28,7 +28,8 @@ import { formatLogCell } from './templates/logs-table.js'; import { PAGE_SIZE, INITIAL_PAGE_SIZE } from './pagination.js'; import { setScrubberPosition, setScrubberRange } from './chart.js'; import { parseUTC } from './chart-state.js'; -import { VirtualTable } from './virtual-table.js'; +// VirtualTable is intentionally NOT used — replaced by bucket-row approach. +// The VirtualTable class remains in virtual-table.js for potential future use. // Cached bucket index built from chart data // eslint-disable-next-line prefer-const -- reassigned in buildBucketIndex/loadLogs @@ -36,6 +37,10 @@ let bucketIndex = null; const TIMESTAMP_RE = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/; +// Bucket-row table constants +const ROW_HEIGHT = 28; +const MAX_TOTAL_HEIGHT = 10_000_000; // 10M pixels cap + /** * Build ordered log column list from available columns. * @param {string[]} allColumns @@ -405,9 +410,10 @@ export function initChartCollapseToggle() { } /** - * renderCell callback for VirtualTable. + * renderCell callback for VirtualTable (unused while bucket-row table is active). * Returns HTML string for a single cell. */ +// eslint-disable-next-line no-unused-vars -- kept for future VirtualTable re-enablement function renderCell(col, value) { const { displayValue, cellClass, colorIndicator } = formatLogCell(col.key, value); const escaped = escapeHtml(displayValue); @@ -556,9 +562,10 @@ function adjustTotalRows(pageIdx, rowCount) { } /** - * getData callback for VirtualTable. + * getData callback for VirtualTable (unused while bucket-row table is active). * Fetches a page of log rows from ClickHouse using cursor-based pagination. */ +// eslint-disable-next-line no-unused-vars -- kept for future VirtualTable re-enablement async function getData(startIdx, count) { const pageIdx = Math.floor(startIdx / PAGE_SIZE); @@ -622,55 +629,190 @@ function destroyVirtualTable() { } } +// Bucket-row table state +let bucketTableContainer = null; +let bucketScrollHandler = null; + +/** + * Compute bucket heights, applying proportional scaling if total exceeds MAX_TOTAL_HEIGHT. + * @param {Array} chartData - array of { t, cnt_ok, cnt_4xx, cnt_5xx } + * @returns {{ buckets: Array<{t: string, count: number, height: number}>, totalHeight: number }} + */ +export function computeBucketHeights(chartData) { + if (!chartData || chartData.length === 0) return { buckets: [], totalHeight: 0 }; + + const buckets = chartData.map((b) => { + const count = (parseInt(b.cnt_ok, 10) || 0) + + (parseInt(b.cnt_4xx, 10) || 0) + + (parseInt(b.cnt_5xx, 10) || 0); + return { t: b.t, count }; + }); + + // Calculate natural heights + let totalHeight = 0; + for (const b of buckets) { + const h = Math.max(b.count, 1) * ROW_HEIGHT; + b.height = h; + totalHeight += h; + } + + // Scale proportionally if over cap + if (totalHeight > MAX_TOTAL_HEIGHT) { + const scale = MAX_TOTAL_HEIGHT / totalHeight; + totalHeight = 0; + for (const b of buckets) { + b.height = Math.max(Math.round(b.height * scale), ROW_HEIGHT); + totalHeight += b.height; + } + } + + return { buckets, totalHeight }; +} + +/** + * Sync the chart scrubber to the visible bucket range. + * Finds which bucket is at the top and bottom of the viewport. + * @param {HTMLElement} scrollContainer + */ +function syncBucketScrubber(scrollContainer) { + const rows = scrollContainer.querySelectorAll('tbody tr.bucket-row'); + if (rows.length === 0) return; + + const { scrollTop } = scrollContainer; + const viewportBottom = scrollTop + scrollContainer.clientHeight; + let firstVisible = null; + let lastVisible = null; + + for (const row of rows) { + const top = row.offsetTop; + const bottom = top + row.offsetHeight; + if (bottom > scrollTop && top < viewportBottom) { + if (!firstVisible) firstVisible = row; + lastVisible = row; + } + // Optimization: stop if we've passed the viewport + if (top > viewportBottom) break; + } + + if (firstVisible && lastVisible) { + const firstTs = firstVisible.id.replace('bucket-', ''); + const lastTs = lastVisible.id.replace('bucket-', ''); + const firstDate = parseUTC(firstTs); + const lastDate = parseUTC(lastTs); + // Rows are newest-first, so first visible is newer + if (firstDate.getTime() !== lastDate.getTime()) { + setScrubberRange(firstDate, lastDate); + } else { + setScrubberPosition(firstDate); + } + } +} + +/** + * Render the bucket-row table from chart data. + * Each chart bucket gets one with proportional height. + * @param {HTMLElement} el - .logs-table-container element + * @param {Array} chartData - state.chartData array + */ +export function renderBucketTable(el, chartData) { + if (!chartData || chartData.length === 0) { + // eslint-disable-next-line no-param-reassign -- DOM manipulation + el.innerHTML = '
No chart data available for bucket table
'; + return; + } + + const { buckets } = computeBucketHeights(chartData); + + // Build table HTML + const numColumns = 7; // placeholder colspan + let tbodyHtml = ''; + + // Reverse to newest-first (chart data is oldest-first) + for (let i = buckets.length - 1; i >= 0; i -= 1) { + const b = buckets[i]; + const rowCount = b.count; + const label = rowCount === 1 ? '1 row' : `${rowCount.toLocaleString()} rows`; + tbodyHtml += `` + + `` + + ''; + } + + // eslint-disable-next-line no-param-reassign -- DOM manipulation + el.innerHTML = `
${label}
+ + ${tbodyHtml} +
Log Buckets
`; + + bucketTableContainer = el; + + // Set up scroll listener for scrubber sync + if (bucketScrollHandler) { + el.removeEventListener('scroll', bucketScrollHandler); + } + bucketScrollHandler = () => { + syncBucketScrubber(el); + }; + el.addEventListener('scroll', bucketScrollHandler, { passive: true }); +} + +/** + * Clean up bucket table event listeners. + */ +function destroyBucketTable() { + if (bucketTableContainer && bucketScrollHandler) { + bucketTableContainer.removeEventListener('scroll', bucketScrollHandler); + } + bucketScrollHandler = null; + bucketTableContainer = null; +} + /** * Create or reconfigure the VirtualTable instance. + * Currently bypassed in favor of the bucket-row table approach. */ function ensureVirtualTable() { const container = logsView.querySelector('.logs-table-container'); if (!container) return; - if (virtualTable) { - // Already exists — just clear cache and re-render - virtualTable.clearCache(); - virtualTable.setTotalRows(0); + // Use bucket-row table when chart data is available + if (state.chartData && state.chartData.length > 0) { + destroyVirtualTable(); + renderBucketTable(container, state.chartData); return; } - // Clear loading placeholder - container.innerHTML = ''; - - virtualTable = new VirtualTable({ - container, - rowHeight: 28, - columns: currentColumns.length > 0 ? buildVirtualColumns(currentColumns) : [], - getData, - renderCell, - onVisibleRangeChange(firstRow, lastRow) { - // Sync chart scrubber to the visible table range - const firstRowData = getRowFromCache(firstRow); - const lastRowData = getRowFromCache(lastRow); - if (firstRowData?.timestamp && lastRowData?.timestamp) { - setScrubberRange(parseUTC(firstRowData.timestamp), parseUTC(lastRowData.timestamp)); - } else { - // Fallback to single point if we don't have both endpoints - const midIdx = Math.floor((firstRow + lastRow) / 2); - const row = getRowFromCache(midIdx); - if (row?.timestamp) { - setScrubberPosition(parseUTC(row.timestamp)); - } - } - }, - onRowClick(idx, row) { - openLogDetailModal(idx, row); - }, - }); + // Fallback: show loading state when chart data not yet available + container.innerHTML = '
Loading\u2026
'; } // Scroll log table to the row closest to a given timestamp export function scrollLogsToTimestamp(timestamp) { - if (!state.showLogs || !virtualTable) return; + if (!state.showLogs) return; const targetMs = timestamp instanceof Date ? timestamp.getTime() : timestamp; - virtualTable.scrollToTimestamp(targetMs, (row) => parseUTC(row.timestamp).getTime()); + + // Bucket-row approach: find the closest bucket by timestamp + if (bucketTableContainer) { + const rows = bucketTableContainer.querySelectorAll('tbody tr.bucket-row'); + let bestRow = null; + let bestDiff = Infinity; + for (const row of rows) { + const ts = row.id.replace('bucket-', ''); + const diff = Math.abs(parseUTC(ts).getTime() - targetMs); + if (diff < bestDiff) { + bestDiff = diff; + bestRow = row; + } + } + if (bestRow) { + bestRow.scrollIntoView({ behavior: 'smooth', block: 'center' }); + } + return; + } + + // Legacy VirtualTable fallback + if (virtualTable) { + virtualTable.scrollToTimestamp(targetMs, (row) => parseUTC(row.timestamp).getTime()); + } } export function setLogsElements(view, toggleBtn, filtersViewEl) { @@ -725,9 +867,10 @@ function buildBucketIndex(chartData) { } /** - * Estimate total rows from chart data bucket counts. + * Estimate total rows from chart data bucket counts (unused while bucket-row table is active). * @returns {number} */ +// eslint-disable-next-line no-unused-vars -- kept for future VirtualTable re-enablement function estimateTotalRows() { if (!state.chartData || state.chartData.length === 0) return 0; let total = 0; @@ -755,7 +898,7 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) const container = logsView.querySelector('.logs-table-container'); container.classList.add('updating'); - // Set up virtual table + // Render bucket table from chart data (available before log data) ensureVirtualTable(); const timeFilter = getTimeFilter(); @@ -781,30 +924,24 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) state.logsData = rows; state.logsReady = true; - if (rows.length === 0) { + if (rows.length === 0 && (!state.chartData || state.chartData.length === 0)) { container.innerHTML = '
No logs matching current filters
'; + destroyBucketTable(); destroyVirtualTable(); return; } - // Populate initial page cache + // Populate initial page cache (for detail modals) const cursor = rows.length > 0 ? rows[rows.length - 1].timestamp : null; pageCache.set(0, { rows, cursor }); // Set columns from data - currentColumns = getLogColumns(Object.keys(rows[0])); - if (virtualTable) { - virtualTable.setColumns(buildVirtualColumns(currentColumns)); - - // Seed VirtualTable cache with pre-fetched page 0 to avoid re-fetch - virtualTable.seedCache(0, rows); - - // Build bucket index from chart data for accurate scroll mapping - bucketIndex = buildBucketIndex(state.chartData); - const estimated = bucketIndex ? bucketIndex.totalRows : estimateTotalRows(); - const total = estimated > rows.length ? estimated : rows.length * 10; - virtualTable.setTotalRows(total); + if (rows.length > 0) { + currentColumns = getLogColumns(Object.keys(rows[0])); } + + // Build bucket index from chart data + bucketIndex = buildBucketIndex(state.chartData); } catch (err) { if (!isCurrent() || isAbortError(err)) return; // eslint-disable-next-line no-console @@ -832,21 +969,12 @@ export function toggleLogsView(saveStateToURL, scrollToTimestamp) { chartSection.classList.add('chart-collapsed'); updateCollapseToggleLabel(); } + // Render bucket table or trigger fresh load ensureVirtualTable(); - // Re-seed virtual table from page cache, or trigger a fresh load if (state.logsReady && pageCache.size > 0) { - const page0 = pageCache.get(0); - if (page0 && page0.rows.length > 0) { - currentColumns = getLogColumns(Object.keys(page0.rows[0])); - virtualTable.setColumns(buildVirtualColumns(currentColumns)); - virtualTable.seedCache(0, page0.rows); - bucketIndex = buildBucketIndex(state.chartData); - const estimated = bucketIndex ? bucketIndex.totalRows : estimateTotalRows(); - const total = estimated > page0.rows.length ? estimated : page0.rows.length * 10; - virtualTable.setTotalRows(total); - if (scrollToTimestamp) { - scrollLogsToTimestamp(scrollToTimestamp); - } + bucketIndex = buildBucketIndex(state.chartData); + if (scrollToTimestamp) { + scrollLogsToTimestamp(scrollToTimestamp); } } else { loadLogs(); @@ -860,7 +988,8 @@ export function toggleLogsView(saveStateToURL, scrollToTimestamp) { if (onShowFiltersView) { requestAnimationFrame(() => onShowFiltersView()); } - // Clean up virtual table when leaving logs view + // Clean up when leaving logs view + destroyBucketTable(); destroyVirtualTable(); } saveStateToURL(); From 6b7d8db888089fcb89366176060f5ca9e68751e2 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 15:31:39 +0100 Subject: [PATCH 19/28] test: add bucket-row table unit tests Tests for computeBucketHeights and renderBucketTable functions: - Bucket height calculation from chart data - Proportional scaling when total exceeds 10M pixels - Table rendering with correct structure and IDs Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/bucket-table.test.js | 184 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 js/bucket-table.test.js diff --git a/js/bucket-table.test.js b/js/bucket-table.test.js new file mode 100644 index 0000000..55905c6 --- /dev/null +++ b/js/bucket-table.test.js @@ -0,0 +1,184 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { assert } from 'chai'; +import { computeBucketHeights, renderBucketTable } from './logs.js'; + +function makeChartData(count, baseCnt = 10) { + return Array.from({ length: count }, (_, i) => ({ + t: `2026-01-15 ${String(Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}:00.000`, + cnt_ok: String(baseCnt), + cnt_4xx: '1', + cnt_5xx: '0', + })); +} + +describe('computeBucketHeights', () => { + it('returns empty for null/empty chart data', () => { + assert.deepEqual(computeBucketHeights(null), { buckets: [], totalHeight: 0 }); + assert.deepEqual(computeBucketHeights([]), { buckets: [], totalHeight: 0 }); + }); + + it('computes height proportional to row count', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '2', cnt_5xx: '1', + }, + { + t: '2026-01-15 00:01:00.000', cnt_ok: '5', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + const { buckets } = computeBucketHeights(data); + assert.strictEqual(buckets.length, 2); + assert.strictEqual(buckets[0].count, 13); + assert.strictEqual(buckets[0].height, 13 * 28); + assert.strictEqual(buckets[1].count, 5); + assert.strictEqual(buckets[1].height, 5 * 28); + }); + + it('enforces minimum height of 28px for empty buckets', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '0', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + const { buckets } = computeBucketHeights(data); + assert.strictEqual(buckets[0].count, 0); + assert.strictEqual(buckets[0].height, 28); // min 1 * ROW_HEIGHT + }); + + it('scales heights when total exceeds 10M pixels', () => { + // Create buckets that would exceed 10M: 100 buckets * 5000 rows * 28px = 14M + const data = Array.from({ length: 100 }, (_, i) => ({ + t: `2026-01-15 ${String(Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}:00.000`, + cnt_ok: '5000', + cnt_4xx: '0', + cnt_5xx: '0', + })); + const { buckets, totalHeight } = computeBucketHeights(data); + assert.ok(totalHeight <= 10_000_000 + 100 * 28, 'total height should be capped near 10M'); + // All buckets should still have at least ROW_HEIGHT + for (const b of buckets) { + assert.ok(b.height >= 28, 'each bucket should have at least 28px height'); + } + }); +}); + +describe('renderBucketTable', () => { + let container; + + beforeEach(() => { + container = document.createElement('div'); + document.body.appendChild(container); + }); + + afterEach(() => { + if (container.parentNode) container.parentNode.removeChild(container); + }); + + it('renders empty message for null chart data', () => { + renderBucketTable(container, null); + assert.include(container.textContent, 'No chart data'); + }); + + it('renders correct number of bucket rows', () => { + const data = makeChartData(5); + renderBucketTable(container, data); + const rows = container.querySelectorAll('tbody tr.bucket-row'); + assert.strictEqual(rows.length, 5); + }); + + it('renders rows in newest-first order', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '0', cnt_5xx: '0', + }, + { + t: '2026-01-15 00:01:00.000', cnt_ok: '20', cnt_4xx: '0', cnt_5xx: '0', + }, + { + t: '2026-01-15 00:02:00.000', cnt_ok: '30', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const rows = container.querySelectorAll('tbody tr.bucket-row'); + // First row should be the newest (last in chart data) + assert.strictEqual(rows[0].id, 'bucket-2026-01-15 00:02:00.000'); + assert.strictEqual(rows[1].id, 'bucket-2026-01-15 00:01:00.000'); + assert.strictEqual(rows[2].id, 'bucket-2026-01-15 00:00:00.000'); + }); + + it('each row has correct id attribute', () => { + const data = [ + { + t: '2026-01-15 12:30:00.000', cnt_ok: '5', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const row = container.querySelector('tbody tr.bucket-row'); + assert.strictEqual(row.id, 'bucket-2026-01-15 12:30:00.000'); + }); + + it('each row has proportional height', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '0', cnt_5xx: '0', + }, + { + t: '2026-01-15 00:01:00.000', cnt_ok: '20', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const rows = container.querySelectorAll('tbody tr.bucket-row'); + // Newest first: row 0 = 20 rows, row 1 = 10 rows + assert.strictEqual(rows[0].style.height, `${20 * 28}px`); + assert.strictEqual(rows[1].style.height, `${10 * 28}px`); + }); + + it('displays row count in placeholder text', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '100', cnt_4xx: '5', cnt_5xx: '2', + }, + ]; + renderBucketTable(container, data); + const td = container.querySelector('.bucket-placeholder'); + assert.include(td.textContent, '107'); + assert.include(td.textContent, 'rows'); + }); + + it('uses singular "row" for count of 1', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '1', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const td = container.querySelector('.bucket-placeholder'); + assert.strictEqual(td.textContent, '1 row'); + }); + + it('renders table with sticky header', () => { + const data = makeChartData(3); + renderBucketTable(container, data); + const thead = container.querySelector('thead'); + assert.ok(thead, 'table should have thead'); + const th = thead.querySelector('th'); + assert.ok(th, 'thead should have th'); + }); + + it('renders table with logs-table class', () => { + const data = makeChartData(3); + renderBucketTable(container, data); + const table = container.querySelector('table.logs-table'); + assert.ok(table, 'table should have logs-table class'); + }); +}); From 63557be68bc116a7c6d90dcddba3ca285e289a8c Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 16:04:28 +0100 Subject: [PATCH 20/28] fix: render bucket table when chart data arrives after loadLogs When the user starts in logs view, loadLogs() runs before the chart query. ensureVirtualTable() finds no chart data and shows "Loading..." which never updates. Fix by adding a tryRenderBucketTable() callback that fires when chart data arrives, and also at end of loadLogs(). - Add tryRenderBucketTable() in logs.js (exported, idempotent) - Add setOnChartDataReady() callback in chart.js - Wire callback in dashboard-init.js Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/chart.js | 13 +++++++++++++ js/dashboard-init.js | 7 ++++++- js/logs.js | 20 ++++++++++++-------- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/js/chart.js b/js/chart.js index f893aa7..402f5ca 100644 --- a/js/chart.js +++ b/js/chart.js @@ -100,6 +100,17 @@ let scrubberRangeEnd = null; // eslint-disable-line prefer-const let onChartHoverTimestamp = null; let onChartClickTimestamp = null; +// Callback invoked when chart data becomes available (set by dashboard-init.js) +let onChartDataReady = null; + +/** + * Register a callback to be invoked when chart data is set. + * @param {Function} callback + */ +export function setOnChartDataReady(callback) { + onChartDataReady = callback; +} + /** * Set callback for chart hover → scroll sync * @param {Function} callback - Called with timestamp when hovering chart in logs view @@ -929,6 +940,8 @@ export async function loadTimeSeries(requestContext = getRequestContext('dashboa if (!isCurrent()) return; state.chartData = result.data; renderChart(result.data); + // Notify logs view that chart data is available (bucket table may need rendering) + if (onChartDataReady) onChartDataReady(); } catch (err) { if (!isCurrent() || isAbortError(err)) return; // eslint-disable-next-line no-console diff --git a/js/dashboard-init.js b/js/dashboard-init.js index 549a1b6..30da94d 100644 --- a/js/dashboard-init.js +++ b/js/dashboard-init.js @@ -29,7 +29,7 @@ import { } from './timer.js'; import { loadTimeSeries, setupChartNavigation, getDetectedAnomalies, getLastChartData, - renderChart, setOnChartHoverTimestamp, setOnChartClickTimestamp, + renderChart, setOnChartHoverTimestamp, setOnChartClickTimestamp, setOnChartDataReady, } from './chart.js'; import { loadAllBreakdowns, loadBreakdown, getBreakdowns, markSlowestFacet, resetFacetTimings, @@ -41,6 +41,7 @@ import { } from './filters.js'; import { loadLogs, toggleLogsView, setLogsElements, setOnShowFiltersView, scrollLogsToTimestamp, + tryRenderBucketTable, } from './logs.js'; import { loadHostAutocomplete } from './autocomplete.js'; import { initModal, closeQuickLinksModal } from './modal.js'; @@ -202,6 +203,10 @@ export function initDashboard(config = {}) { } }); + // When chart data arrives, try rendering the bucket table (fixes race condition + // where logs view shows "Loading..." because chart data wasn't available yet) + setOnChartDataReady(() => tryRenderBucketTable()); + // Chart→Scroll sync: throttled to avoid excessive scrolling let lastHoverScroll = 0; setOnChartHoverTimestamp((timestamp) => { diff --git a/js/logs.js b/js/logs.js index 57f3045..18a376c 100644 --- a/js/logs.js +++ b/js/logs.js @@ -28,10 +28,7 @@ import { formatLogCell } from './templates/logs-table.js'; import { PAGE_SIZE, INITIAL_PAGE_SIZE } from './pagination.js'; import { setScrubberPosition, setScrubberRange } from './chart.js'; import { parseUTC } from './chart-state.js'; -// VirtualTable is intentionally NOT used — replaced by bucket-row approach. -// The VirtualTable class remains in virtual-table.js for potential future use. - -// Cached bucket index built from chart data +// VirtualTable intentionally NOT used — replaced by bucket-row approach. // eslint-disable-next-line prefer-const -- reassigned in buildBucketIndex/loadLogs let bucketIndex = null; @@ -88,11 +85,9 @@ let logsView = null; let viewToggleBtn = null; let filtersView = null; -// VirtualTable instance let virtualTable = null; -// Page cache for cursor-based pagination -// Maps pageIndex → { rows, cursor (timestamp of last row) } +// Page cache: pageIndex → { rows, cursor (timestamp of last row) } const pageCache = new Map(); let currentColumns = []; @@ -112,7 +107,6 @@ function showCopyFeedback() { }, 1500); } -// Log detail modal element let logDetailModal = null; /** @@ -785,6 +779,14 @@ function ensureVirtualTable() { container.innerHTML = '
Loading\u2026
'; } +// Re-render bucket table when chart data arrives after loadLogs() (race condition fix) +export function tryRenderBucketTable() { + if (!state.showLogs || !logsView || !state.chartData?.length) return; + const container = logsView.querySelector('.logs-table-container'); + if (!container || container.querySelector('.bucket-table')) return; + ensureVirtualTable(); +} + // Scroll log table to the row closest to a given timestamp export function scrollLogsToTimestamp(timestamp) { if (!state.showLogs) return; @@ -951,6 +953,8 @@ export async function loadLogs(requestContext = getRequestContext('dashboard')) if (isCurrent()) { state.logsLoading = false; container.classList.remove('updating'); + // Chart data may have arrived while the logs query was in-flight + tryRenderBucketTable(); } } } From c023ff9ca56652dcaf1b226f88b90976a72db569 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 16:13:01 +0100 Subject: [PATCH 21/28] style: add zebra striping to bucket-row table for visual row distinction Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/css/logs.css b/css/logs.css index c4df646..d9557ac 100644 --- a/css/logs.css +++ b/css/logs.css @@ -240,6 +240,10 @@ white-space: nowrap; } +.bucket-table .bucket-row:nth-child(even) .bucket-placeholder { + background: rgba(0, 0, 0, 0.02); +} + .bucket-table .bucket-row:hover .bucket-placeholder { background: var(--bg); } From 1305180a67900efde6ea2a9a2cb5332b959f004e Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 16:17:36 +0100 Subject: [PATCH 22/28] style: repeating gradient stripes simulate data rows in bucket placeholders Replace nth-child(even) zebra striping with a repeating-linear-gradient on each bucket placeholder cell. 28px alternating bands match ROW_HEIGHT, creating the visual effect of individual rows scrolling by even though each bucket is a single tall . Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/css/logs.css b/css/logs.css index d9557ac..ca55797 100644 --- a/css/logs.css +++ b/css/logs.css @@ -238,10 +238,13 @@ border-bottom: 1px solid var(--border); padding: 0 12px; white-space: nowrap; -} - -.bucket-table .bucket-row:nth-child(even) .bucket-placeholder { - background: rgba(0, 0, 0, 0.02); + background: repeating-linear-gradient( + to bottom, + transparent 0px, + transparent 28px, + rgba(0, 0, 0, 0.02) 28px, + rgba(0, 0, 0, 0.02) 56px + ); } .bucket-table .bucket-row:hover .bucket-placeholder { From 1d14278201d02995cbb2e4658f48015dfd657a19 Mon Sep 17 00:00:00 2001 From: Lars Trieloff Date: Fri, 13 Feb 2026 16:20:34 +0100 Subject: [PATCH 23/28] fix: remove hover effect on bucket placeholder rows The repeating gradient stripes in bucket rows were being replaced by a solid background on hover, hiding the stripe effect. Agent-Id: agent-1f32891e-559d-4e3f-a729-42a3797046b2 --- css/logs.css | 4 ---- 1 file changed, 4 deletions(-) diff --git a/css/logs.css b/css/logs.css index ca55797..7b95ff6 100644 --- a/css/logs.css +++ b/css/logs.css @@ -247,10 +247,6 @@ ); } -.bucket-table .bucket-row:hover .bucket-placeholder { - background: var(--bg); -} - /* Copy feedback toast */ .copy-feedback { position: fixed; From 28451717fd4189bf9a8d2c164fc07c6c6ee891c8 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 16:29:06 +0100 Subject: [PATCH 24/28] fix: remove range-mode scrubber and use instant scroll for chart-hover sync - Remove setScrubberRange(), restoreScrubberRange(), and range-mode variables from chart.js - Remove .chart-scrubber-line.range-mode CSS rule - Update syncBucketScrubber() to use setScrubberPosition(firstDate) instead of setScrubberRange - Change scrollIntoView from 'smooth' to 'instant' for responsive chart-hover scrolling Signed-off-by: Lars Trieloff --- css/chart.css | 7 ------- js/chart.js | 52 +-------------------------------------------------- js/logs.js | 20 +++++--------------- 3 files changed, 6 insertions(+), 73 deletions(-) diff --git a/css/chart.css b/css/chart.css index 67d34bc..820dd98 100644 --- a/css/chart.css +++ b/css/chart.css @@ -149,13 +149,6 @@ opacity: 0.6; } -.chart-scrubber-line.range-mode { - background: rgba(59, 130, 246, 0.12); - border-left: 2px solid rgba(59, 130, 246, 0.5); - border-right: 2px solid rgba(59, 130, 246, 0.5); - opacity: 1; -} - .chart-scrubber-status { position: absolute; bottom: 0; diff --git a/js/chart.js b/js/chart.js index 402f5ca..787509b 100644 --- a/js/chart.js +++ b/js/chart.js @@ -92,10 +92,6 @@ let isDragging = false; let dragStartX = null; let justCompletedDrag = false; -// Scrubber range state (for table viewport sync) -let scrubberRangeStart = null; // eslint-disable-line prefer-const -let scrubberRangeEnd = null; // eslint-disable-line prefer-const - // Callback for chart→scroll sync (set by logs.js) let onChartHoverTimestamp = null; let onChartClickTimestamp = null; @@ -141,47 +137,9 @@ export function setScrubberPosition(timestamp) { scrubberLine.style.left = `${x}px`; scrubberLine.style.top = `${padding.top}px`; scrubberLine.style.height = `${height - padding.top - padding.bottom}px`; - scrubberLine.style.width = ''; - scrubberLine.classList.remove('range-mode'); scrubberLine.classList.add('visible'); } -/** - * Show the scrubber as a shaded band between two timestamps (table viewport sync) - * @param {Date} startTime - Start of visible range - * @param {Date} endTime - End of visible range - */ -export function setScrubberRange(startTime, endTime) { - if (!scrubberLine) return; - const startX = getXAtTime(startTime); - const endX = getXAtTime(endTime); - const chartLayout = getChartLayout(); - if (!chartLayout) return; - - const { padding, height } = chartLayout; - const { top } = padding; - const bandHeight = height - padding.top - padding.bottom; - - scrubberLine.style.left = `${Math.min(startX, endX)}px`; - scrubberLine.style.width = `${Math.max(2, Math.abs(endX - startX))}px`; - scrubberLine.style.top = `${top}px`; - scrubberLine.style.height = `${bandHeight}px`; - scrubberLine.classList.add('visible', 'range-mode'); - - // Store range so it can be restored after hover - scrubberRangeStart = startTime; - scrubberRangeEnd = endTime; -} - -/** - * Restore the scrubber range band if one was set (called on mouseleave) - */ -function restoreScrubberRange() { - if (scrubberRangeStart && scrubberRangeEnd) { - setScrubberRange(scrubberRangeStart, scrubberRangeEnd); - } -} - /** * Initialize canvas for chart rendering */ @@ -659,9 +617,6 @@ export function setupChartNavigation(callback) { // Show/hide scrubber on container hover container.addEventListener('mouseenter', () => { - // Switch from range band to single-line hover mode - scrubberLine.classList.remove('range-mode'); - scrubberLine.style.width = ''; scrubberLine.classList.add('visible'); scrubberStatusBar.classList.add('visible'); }); @@ -670,12 +625,7 @@ export function setupChartNavigation(callback) { scrubberStatusBar.classList.remove('visible'); hideReleaseTooltip(); canvas.style.cursor = ''; - // Restore range band if logs view is active - if (state.showLogs && scrubberRangeStart) { - restoreScrubberRange(); - } else { - scrubberLine.classList.remove('visible'); - } + scrubberLine.classList.remove('visible'); }); container.addEventListener('mousemove', (e) => { diff --git a/js/logs.js b/js/logs.js index 18a376c..da2262f 100644 --- a/js/logs.js +++ b/js/logs.js @@ -26,7 +26,7 @@ import { import { loadSql } from './sql-loader.js'; import { formatLogCell } from './templates/logs-table.js'; import { PAGE_SIZE, INITIAL_PAGE_SIZE } from './pagination.js'; -import { setScrubberPosition, setScrubberRange } from './chart.js'; +import { setScrubberPosition } from './chart.js'; import { parseUTC } from './chart-state.js'; // VirtualTable intentionally NOT used — replaced by bucket-row approach. // eslint-disable-next-line prefer-const -- reassigned in buildBucketIndex/loadLogs @@ -664,8 +664,7 @@ export function computeBucketHeights(chartData) { } /** - * Sync the chart scrubber to the visible bucket range. - * Finds which bucket is at the top and bottom of the viewport. + * Sync the chart scrubber to the first visible bucket. * @param {HTMLElement} scrollContainer */ function syncBucketScrubber(scrollContainer) { @@ -675,30 +674,21 @@ function syncBucketScrubber(scrollContainer) { const { scrollTop } = scrollContainer; const viewportBottom = scrollTop + scrollContainer.clientHeight; let firstVisible = null; - let lastVisible = null; for (const row of rows) { const top = row.offsetTop; const bottom = top + row.offsetHeight; if (bottom > scrollTop && top < viewportBottom) { if (!firstVisible) firstVisible = row; - lastVisible = row; } // Optimization: stop if we've passed the viewport if (top > viewportBottom) break; } - if (firstVisible && lastVisible) { + if (firstVisible) { const firstTs = firstVisible.id.replace('bucket-', ''); - const lastTs = lastVisible.id.replace('bucket-', ''); const firstDate = parseUTC(firstTs); - const lastDate = parseUTC(lastTs); - // Rows are newest-first, so first visible is newer - if (firstDate.getTime() !== lastDate.getTime()) { - setScrubberRange(firstDate, lastDate); - } else { - setScrubberPosition(firstDate); - } + setScrubberPosition(firstDate); } } @@ -806,7 +796,7 @@ export function scrollLogsToTimestamp(timestamp) { } } if (bestRow) { - bestRow.scrollIntoView({ behavior: 'smooth', block: 'center' }); + bestRow.scrollIntoView({ behavior: 'instant', block: 'center' }); } return; } From e6f616f0e5c629841e2ffbf342fcb0fbbec73a22 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 18:17:56 +0100 Subject: [PATCH 25/28] chore: add benchmark script for bucket fetch response times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures ClickHouse query latency for 8 test cases with varying row limits (100–50000) and time windows (5s–15min) to inform the bucket data loading strategy. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- scripts/benchmark-bucket-fetch.mjs | 250 +++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100755 scripts/benchmark-bucket-fetch.mjs diff --git a/scripts/benchmark-bucket-fetch.mjs b/scripts/benchmark-bucket-fetch.mjs new file mode 100755 index 0000000..1448c78 --- /dev/null +++ b/scripts/benchmark-bucket-fetch.mjs @@ -0,0 +1,250 @@ +#!/usr/bin/env node + +/* + * Copyright 2025 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/** + * Benchmark ClickHouse query response times for different row counts. + * Measures fetch latency for the same columns used by the dashboard logs view. + * + * Usage: node scripts/benchmark-bucket-fetch.mjs + */ + +const CLICKHOUSE_HOST = 's2p5b8wmt5.eastus2.azure.clickhouse.cloud'; +const CLICKHOUSE_PORT = 8443; +const DATABASE = 'helix_logs_production'; +const RUNS_PER_TEST = 3; +const DELAY_BETWEEN_RUNS_MS = 500; + +const COLUMNS = [ + '`timestamp`', + '`source`', + '`response.status`', + '`request.method`', + '`request.host`', + '`request.url`', + '`cdn.cache_status`', + '`response.headers.content_type`', + '`helix.request_type`', + '`helix.backend_type`', + '`request.headers.x_forwarded_host`', + '`request.headers.referer`', + '`request.headers.user_agent`', + '`client.ip`', + '`request.headers.x_forwarded_for`', + '`response.headers.x_error`', + '`request.headers.accept`', + '`request.headers.accept_encoding`', + '`request.headers.cache_control`', + '`request.headers.x_byo_cdn_type`', + '`response.headers.location`', +].join(', '); + +const TABLE = `${DATABASE}.cdn_requests_v2`; + +const TEST_CASES = [ + { + name: 'LIMIT 100 (initial viewport)', + limit: 100, + timeWindow: '15 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 15 MINUTE +ORDER BY timestamp DESC LIMIT 100`, + }, + { + name: 'LIMIT 500 (one page)', + limit: 500, + timeWindow: '15 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 15 MINUTE +ORDER BY timestamp DESC LIMIT 500`, + }, + { + name: 'LIMIT 2000 (medium bucket)', + limit: 2000, + timeWindow: '15 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 15 MINUTE +ORDER BY timestamp DESC LIMIT 2000`, + }, + { + name: 'LIMIT 6000 (full large bucket)', + limit: 6000, + timeWindow: '15 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 15 MINUTE +ORDER BY timestamp DESC LIMIT 6000`, + }, + { + name: '5-second bucket, all rows', + limit: 10000, + timeWindow: '5 sec', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 5 SECOND +ORDER BY timestamp DESC LIMIT 10000`, + }, + { + name: '5-min bucket, LIMIT 500', + limit: 500, + timeWindow: '5 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 5 MINUTE +ORDER BY timestamp DESC LIMIT 500`, + }, + { + name: '5-min bucket, LIMIT 5000', + limit: 5000, + timeWindow: '5 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 5 MINUTE +ORDER BY timestamp DESC LIMIT 5000`, + }, + { + name: '5-min bucket, LIMIT 50000', + limit: 50000, + timeWindow: '5 min', + sql: `SELECT ${COLUMNS} FROM ${TABLE} +WHERE timestamp >= now() - INTERVAL 5 MINUTE +ORDER BY timestamp DESC LIMIT 50000`, + }, +]; + +function printUsage() { + console.log('Usage: node scripts/benchmark-bucket-fetch.mjs '); + console.log(''); + console.log('Benchmarks ClickHouse query response times for different row counts'); + console.log('using the same columns as the dashboard logs view.'); + console.log(''); + console.log('Arguments:'); + console.log(' username ClickHouse username'); + console.log(' password ClickHouse password'); +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) { + return (sorted[mid - 1] + sorted[mid]) / 2; + } + return sorted[mid]; +} + +function sleep(ms) { + // eslint-disable-next-line no-promise-executor-return + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +async function runQuery(sql, auth, runIndex) { + const taggedSql = `/* run=${runIndex} t=${Date.now()} */ ${sql}`; + const url = `https://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/?database=${DATABASE}&use_query_cache=0`; + + const start = performance.now(); + const resp = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Basic ${Buffer.from(auth).toString('base64')}`, + 'Content-Type': 'text/plain', + }, + body: `${taggedSql} FORMAT JSON`, + }); + + if (!resp.ok) { + const body = await resp.text(); + throw new Error(`ClickHouse HTTP ${resp.status}: ${body.slice(0, 200)}`); + } + + const json = await resp.json(); + const elapsed = performance.now() - start; + + return { + elapsed, + rows: json.rows, + transferBytes: JSON.stringify(json.data).length, + serverElapsed: json.statistics?.elapsed ?? null, + }; +} + +async function main() { + const args = process.argv.slice(2); + + if (args.includes('--help') || args.includes('-h')) { + printUsage(); + process.exit(0); + } + + if (args.length < 2) { + console.error('Error: username and password are required.\n'); + printUsage(); + process.exit(1); + } + + const [username, password] = args; + const auth = `${username}:${password}`; + + console.log(`Running benchmark against ${CLICKHOUSE_HOST} as ${username}`); + console.log(`${RUNS_PER_TEST} runs per test, reporting median\n`); + + const results = []; + + for (const testCase of TEST_CASES) { + const timings = []; + let lastResult = null; + + process.stdout.write(` ${testCase.name} ...`); + + // eslint-disable-next-line no-await-in-loop -- sequential runs required for stable timings + for (let i = 0; i < RUNS_PER_TEST; i += 1) { + if (i > 0) { + // eslint-disable-next-line no-await-in-loop + await sleep(DELAY_BETWEEN_RUNS_MS); + } + // eslint-disable-next-line no-await-in-loop + const result = await runQuery(testCase.sql, auth, i); + timings.push(result.elapsed); + lastResult = result; + } + + const medianTime = median(timings); + const transferKb = (lastResult.transferBytes / 1024).toFixed(1); + + process.stdout.write(` ${medianTime.toFixed(0)} ms (${lastResult.rows} rows, ${transferKb} KB)\n`); + + results.push({ + name: testCase.name, + limit: testCase.limit, + timeWindow: testCase.timeWindow, + rows: lastResult.rows, + medianMs: medianTime, + transferKb: parseFloat(transferKb), + serverElapsedMs: lastResult.serverElapsed !== null + ? (lastResult.serverElapsed * 1000).toFixed(0) + : 'n/a', + }); + } + + console.log('\n## Results\n'); + console.log('| # | Test | Limit | Time Window | Rows Returned | Median Time (ms) | Server Time (ms) | Transfer Size (KB) |'); + console.log('|---|------|-------|-------------|---------------|-------------------|-------------------|--------------------|'); + + results.forEach((r, i) => { + console.log( + `| ${i + 1} | ${r.name} | ${r.limit} | ${r.timeWindow} | ${r.rows} | ${r.medianMs.toFixed(0)} | ${r.serverElapsedMs} | ${r.transferKb} |`, + ); + }); +} + +main().catch((err) => { + console.error('Benchmark failed:', err.message); + process.exit(1); +}); From de84e32d90c116a9b1c0d2116619ae6982667faa Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 18:44:44 +0100 Subject: [PATCH 26/28] feat: split bucket placeholders into head and tail rows Each bucket with > 500 rows now renders two elements: a head row (500 rows worth of height) and a tail row (remaining rows). Buckets with <= 500 rows render a single head row as before. This prepares for Wave 14 where the head placeholder will be replaced with actual data rows from a fast LIMIT 500 query, and the tail placeholder from a slower background query. - computeBucketHeights() returns headCount/tailCount/headHeight/tailHeight - renderBucketTable() renders head+tail with distinct labels - Scrubber sync and scrollToTimestamp updated for bucket-head-{ts} IDs - 819 tests pass, lint clean Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/bucket-table.test.js | 195 +++++++++++++++++++++++++++++++++++----- js/logs.js | 63 +++++++------ 2 files changed, 209 insertions(+), 49 deletions(-) diff --git a/js/bucket-table.test.js b/js/bucket-table.test.js index 55905c6..ea5f1c0 100644 --- a/js/bucket-table.test.js +++ b/js/bucket-table.test.js @@ -27,7 +27,7 @@ describe('computeBucketHeights', () => { assert.deepEqual(computeBucketHeights([]), { buckets: [], totalHeight: 0 }); }); - it('computes height proportional to row count', () => { + it('computes headHeight proportional to row count for small buckets', () => { const data = [ { t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '2', cnt_5xx: '1', @@ -39,12 +39,14 @@ describe('computeBucketHeights', () => { const { buckets } = computeBucketHeights(data); assert.strictEqual(buckets.length, 2); assert.strictEqual(buckets[0].count, 13); - assert.strictEqual(buckets[0].height, 13 * 28); + assert.strictEqual(buckets[0].headHeight, 13 * 28); + assert.strictEqual(buckets[0].tailHeight, 0); assert.strictEqual(buckets[1].count, 5); - assert.strictEqual(buckets[1].height, 5 * 28); + assert.strictEqual(buckets[1].headHeight, 5 * 28); + assert.strictEqual(buckets[1].tailHeight, 0); }); - it('enforces minimum height of 28px for empty buckets', () => { + it('enforces minimum headHeight of 28px for empty buckets', () => { const data = [ { t: '2026-01-15 00:00:00.000', cnt_ok: '0', cnt_4xx: '0', cnt_5xx: '0', @@ -52,7 +54,8 @@ describe('computeBucketHeights', () => { ]; const { buckets } = computeBucketHeights(data); assert.strictEqual(buckets[0].count, 0); - assert.strictEqual(buckets[0].height, 28); // min 1 * ROW_HEIGHT + assert.strictEqual(buckets[0].headHeight, 28); // min 1 * ROW_HEIGHT + assert.strictEqual(buckets[0].tailHeight, 0); }); it('scales heights when total exceeds 10M pixels', () => { @@ -64,12 +67,60 @@ describe('computeBucketHeights', () => { cnt_5xx: '0', })); const { buckets, totalHeight } = computeBucketHeights(data); - assert.ok(totalHeight <= 10_000_000 + 100 * 28, 'total height should be capped near 10M'); - // All buckets should still have at least ROW_HEIGHT + assert.ok(totalHeight <= 10_000_000 + 200 * 28, 'total height should be capped near 10M'); + // All buckets should still have at least ROW_HEIGHT for head for (const b of buckets) { - assert.ok(b.height >= 28, 'each bucket should have at least 28px height'); + assert.ok(b.headHeight >= 28, 'each bucket head should have at least 28px height'); } }); + + it('bucket with count <= 500 has headCount = count and tailCount = 0', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '200', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + const { buckets } = computeBucketHeights(data); + assert.strictEqual(buckets[0].headCount, 200); + assert.strictEqual(buckets[0].tailCount, 0); + assert.strictEqual(buckets[0].headHeight, 200 * 28); + assert.strictEqual(buckets[0].tailHeight, 0); + }); + + it('bucket with count > 500 has headCount = 500 and tailCount = count - 500', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '1000', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + const { buckets } = computeBucketHeights(data); + assert.strictEqual(buckets[0].headCount, 500); + assert.strictEqual(buckets[0].tailCount, 500); + assert.strictEqual(buckets[0].headHeight, 500 * 28); + assert.strictEqual(buckets[0].tailHeight, 500 * 28); + }); + + it('head + tail height equals total bucket height', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '800', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + const { buckets } = computeBucketHeights(data); + assert.strictEqual(buckets[0].headHeight + buckets[0].tailHeight, 800 * 28); + }); + + it('bucket with exactly 500 rows has no tail', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '500', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + const { buckets } = computeBucketHeights(data); + assert.strictEqual(buckets[0].headCount, 500); + assert.strictEqual(buckets[0].tailCount, 0); + assert.strictEqual(buckets[0].tailHeight, 0); + }); }); describe('renderBucketTable', () => { @@ -89,14 +140,17 @@ describe('renderBucketTable', () => { assert.include(container.textContent, 'No chart data'); }); - it('renders correct number of bucket rows', () => { + it('renders correct number of head rows for small buckets', () => { const data = makeChartData(5); renderBucketTable(container, data); - const rows = container.querySelectorAll('tbody tr.bucket-row'); - assert.strictEqual(rows.length, 5); + const headRows = container.querySelectorAll('tbody tr.bucket-head'); + assert.strictEqual(headRows.length, 5); + // No tail rows (baseCnt 10+1=11 < 500) + const tailRows = container.querySelectorAll('tbody tr.bucket-tail'); + assert.strictEqual(tailRows.length, 0); }); - it('renders rows in newest-first order', () => { + it('renders head rows in newest-first order', () => { const data = [ { t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '0', cnt_5xx: '0', @@ -109,25 +163,25 @@ describe('renderBucketTable', () => { }, ]; renderBucketTable(container, data); - const rows = container.querySelectorAll('tbody tr.bucket-row'); + const rows = container.querySelectorAll('tbody tr.bucket-head'); // First row should be the newest (last in chart data) - assert.strictEqual(rows[0].id, 'bucket-2026-01-15 00:02:00.000'); - assert.strictEqual(rows[1].id, 'bucket-2026-01-15 00:01:00.000'); - assert.strictEqual(rows[2].id, 'bucket-2026-01-15 00:00:00.000'); + assert.strictEqual(rows[0].id, 'bucket-head-2026-01-15 00:02:00.000'); + assert.strictEqual(rows[1].id, 'bucket-head-2026-01-15 00:01:00.000'); + assert.strictEqual(rows[2].id, 'bucket-head-2026-01-15 00:00:00.000'); }); - it('each row has correct id attribute', () => { + it('each head row has correct id attribute', () => { const data = [ { t: '2026-01-15 12:30:00.000', cnt_ok: '5', cnt_4xx: '0', cnt_5xx: '0', }, ]; renderBucketTable(container, data); - const row = container.querySelector('tbody tr.bucket-row'); - assert.strictEqual(row.id, 'bucket-2026-01-15 12:30:00.000'); + const row = container.querySelector('tbody tr.bucket-head'); + assert.strictEqual(row.id, 'bucket-head-2026-01-15 12:30:00.000'); }); - it('each row has proportional height', () => { + it('each head row has proportional height for small buckets', () => { const data = [ { t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '0', cnt_5xx: '0', @@ -137,13 +191,13 @@ describe('renderBucketTable', () => { }, ]; renderBucketTable(container, data); - const rows = container.querySelectorAll('tbody tr.bucket-row'); + const rows = container.querySelectorAll('tbody tr.bucket-head'); // Newest first: row 0 = 20 rows, row 1 = 10 rows assert.strictEqual(rows[0].style.height, `${20 * 28}px`); assert.strictEqual(rows[1].style.height, `${10 * 28}px`); }); - it('displays row count in placeholder text', () => { + it('displays row count in placeholder text for small bucket', () => { const data = [ { t: '2026-01-15 00:00:00.000', cnt_ok: '100', cnt_4xx: '5', cnt_5xx: '2', @@ -181,4 +235,101 @@ describe('renderBucketTable', () => { const table = container.querySelector('table.logs-table'); assert.ok(table, 'table should have logs-table class'); }); + + it('bucket with > 500 rows produces head and tail rows', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '1000', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const headRows = container.querySelectorAll('tbody tr.bucket-head'); + const tailRows = container.querySelectorAll('tbody tr.bucket-tail'); + assert.strictEqual(headRows.length, 1); + assert.strictEqual(tailRows.length, 1); + assert.strictEqual(headRows[0].id, 'bucket-head-2026-01-15 00:00:00.000'); + assert.strictEqual(tailRows[0].id, 'bucket-tail-2026-01-15 00:00:00.000'); + }); + + it('bucket with <= 500 rows produces only a head row', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '200', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const headRows = container.querySelectorAll('tbody tr.bucket-head'); + const tailRows = container.querySelectorAll('tbody tr.bucket-tail'); + assert.strictEqual(headRows.length, 1); + assert.strictEqual(tailRows.length, 0); + }); + + it('head row height = min(count, 500) * ROW_HEIGHT', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '800', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const headRow = container.querySelector('tbody tr.bucket-head'); + assert.strictEqual(headRow.style.height, `${500 * 28}px`); + }); + + it('tail row height = (count - 500) * ROW_HEIGHT', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '800', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const tailRow = container.querySelector('tbody tr.bucket-tail'); + assert.strictEqual(tailRow.style.height, `${300 * 28}px`); + }); + + it('head label says "500 of {count} rows" when there is a tail', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '1000', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const headTd = container.querySelector('tbody tr.bucket-head .bucket-placeholder'); + assert.strictEqual(headTd.textContent, '500 of 1,000 rows'); + }); + + it('head label says "{count} rows" when there is no tail', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '200', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const headTd = container.querySelector('tbody tr.bucket-head .bucket-placeholder'); + assert.strictEqual(headTd.textContent, '200 rows'); + }); + + it('tail label says "{tailCount} remaining rows"', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '1000', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const tailTd = container.querySelector('tbody tr.bucket-tail .bucket-placeholder'); + assert.strictEqual(tailTd.textContent, '500 remaining rows'); + }); + + it('total height is preserved (head + tail = original single row height)', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '800', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const headRow = container.querySelector('tbody tr.bucket-head'); + const tailRow = container.querySelector('tbody tr.bucket-tail'); + const headH = parseInt(headRow.style.height, 10); + const tailH = parseInt(tailRow.style.height, 10); + assert.strictEqual(headH + tailH, 800 * 28); + }); }); diff --git a/js/logs.js b/js/logs.js index da2262f..bf57852 100644 --- a/js/logs.js +++ b/js/logs.js @@ -628,9 +628,7 @@ let bucketTableContainer = null; let bucketScrollHandler = null; /** - * Compute bucket heights, applying proportional scaling if total exceeds MAX_TOTAL_HEIGHT. - * @param {Array} chartData - array of { t, cnt_ok, cnt_4xx, cnt_5xx } - * @returns {{ buckets: Array<{t: string, count: number, height: number}>, totalHeight: number }} + * Compute bucket heights with head/tail split, scaling if total exceeds MAX_TOTAL_HEIGHT. */ export function computeBucketHeights(chartData) { if (!chartData || chartData.length === 0) return { buckets: [], totalHeight: 0 }; @@ -639,15 +637,19 @@ export function computeBucketHeights(chartData) { const count = (parseInt(b.cnt_ok, 10) || 0) + (parseInt(b.cnt_4xx, 10) || 0) + (parseInt(b.cnt_5xx, 10) || 0); - return { t: b.t, count }; + const headCount = Math.min(count, 500); + const tailCount = Math.max(count - 500, 0); + return { + t: b.t, count, headCount, tailCount, + }; }); // Calculate natural heights let totalHeight = 0; for (const b of buckets) { - const h = Math.max(b.count, 1) * ROW_HEIGHT; - b.height = h; - totalHeight += h; + b.headHeight = Math.max(b.headCount, 1) * ROW_HEIGHT; + b.tailHeight = b.tailCount * ROW_HEIGHT; + totalHeight += b.headHeight + b.tailHeight; } // Scale proportionally if over cap @@ -655,20 +657,18 @@ export function computeBucketHeights(chartData) { const scale = MAX_TOTAL_HEIGHT / totalHeight; totalHeight = 0; for (const b of buckets) { - b.height = Math.max(Math.round(b.height * scale), ROW_HEIGHT); - totalHeight += b.height; + b.headHeight = Math.max(Math.round(b.headHeight * scale), ROW_HEIGHT); + b.tailHeight = b.tailCount > 0 ? Math.max(Math.round(b.tailHeight * scale), ROW_HEIGHT) : 0; + totalHeight += b.headHeight + b.tailHeight; } } return { buckets, totalHeight }; } -/** - * Sync the chart scrubber to the first visible bucket. - * @param {HTMLElement} scrollContainer - */ +/** Sync the chart scrubber to the first visible bucket head row. */ function syncBucketScrubber(scrollContainer) { - const rows = scrollContainer.querySelectorAll('tbody tr.bucket-row'); + const rows = scrollContainer.querySelectorAll('tbody tr.bucket-head'); if (rows.length === 0) return; const { scrollTop } = scrollContainer; @@ -686,18 +686,13 @@ function syncBucketScrubber(scrollContainer) { } if (firstVisible) { - const firstTs = firstVisible.id.replace('bucket-', ''); + const firstTs = firstVisible.id.replace('bucket-head-', ''); const firstDate = parseUTC(firstTs); setScrubberPosition(firstDate); } } -/** - * Render the bucket-row table from chart data. - * Each chart bucket gets one with proportional height. - * @param {HTMLElement} el - .logs-table-container element - * @param {Array} chartData - state.chartData array - */ +/** Render the bucket-row table with head/tail split from chart data. */ export function renderBucketTable(el, chartData) { if (!chartData || chartData.length === 0) { // eslint-disable-next-line no-param-reassign -- DOM manipulation @@ -714,11 +709,25 @@ export function renderBucketTable(el, chartData) { // Reverse to newest-first (chart data is oldest-first) for (let i = buckets.length - 1; i >= 0; i -= 1) { const b = buckets[i]; - const rowCount = b.count; - const label = rowCount === 1 ? '1 row' : `${rowCount.toLocaleString()} rows`; - tbodyHtml += `` - + `${label}` + + // Head row (always present) + let headLabel; + if (b.tailCount > 0) { + headLabel = `500 of ${b.count.toLocaleString()} rows`; + } else { + headLabel = b.count === 1 ? '1 row' : `${b.count.toLocaleString()} rows`; + } + tbodyHtml += `` + + `${headLabel}` + ''; + + // Tail row (only when bucket has > 500 rows) + if (b.tailCount > 0) { + const tailLabel = `${b.tailCount.toLocaleString()} remaining rows`; + tbodyHtml += `` + + `${tailLabel}` + + ''; + } } // eslint-disable-next-line no-param-reassign -- DOM manipulation @@ -784,11 +793,11 @@ export function scrollLogsToTimestamp(timestamp) { // Bucket-row approach: find the closest bucket by timestamp if (bucketTableContainer) { - const rows = bucketTableContainer.querySelectorAll('tbody tr.bucket-row'); + const rows = bucketTableContainer.querySelectorAll('tbody tr.bucket-head'); let bestRow = null; let bestDiff = Infinity; for (const row of rows) { - const ts = row.id.replace('bucket-', ''); + const ts = row.id.replace('bucket-head-', ''); const diff = Math.abs(parseUTC(ts).getTime() - targetMs); if (diff < bestDiff) { bestDiff = diff; From dd7b0976912e3a5b7808cfb3fdcb170a48419f8d Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 19:40:17 +0100 Subject: [PATCH 27/28] feat: load actual log data per bucket via IntersectionObserver When a bucket placeholder scrolls into the viewport, fire per-bucket ClickHouse queries scoped to the bucket's time window. Head placeholder gets LIMIT 500 rows, tail gets LIMIT tailCount OFFSET 500. Both fire in parallel with a concurrency cap of 4. - Create js/bucket-loader.js with IntersectionObserver, fetchBucketRows, replacePlaceholder, concurrency limiter, and abort support - Replace "Log Buckets" header with actual column headers - Style data rows at 28px height matching ROW_HEIGHT - Abort in-flight requests on time range change / view switch - 822 tests pass, lint clean Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- css/logs.css | 7 ++ js/bucket-loader.js | 234 ++++++++++++++++++++++++++++++++++++++++ js/bucket-table.test.js | 32 ++++++ js/logs.js | 68 ++++++------ 4 files changed, 307 insertions(+), 34 deletions(-) create mode 100644 js/bucket-loader.js diff --git a/css/logs.css b/css/logs.css index 7b95ff6..23e8f1d 100644 --- a/css/logs.css +++ b/css/logs.css @@ -247,6 +247,13 @@ ); } +/* Bucket table data rows (loaded on demand) */ +.bucket-table tr[data-row-idx] td { + height: 28px; + padding: 4px 8px; + box-sizing: border-box; +} + /* Copy feedback toast */ .copy-feedback { position: fixed; diff --git a/js/bucket-loader.js b/js/bucket-loader.js new file mode 100644 index 0000000..56c53b1 --- /dev/null +++ b/js/bucket-loader.js @@ -0,0 +1,234 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ + +/** + * Bucket data loader — extracted from logs.js to stay within the + * max-lines lint limit. Handles per-bucket on-demand data fetching, + * IntersectionObserver setup, and placeholder replacement. + */ + +import { DATABASE } from './config.js'; +import { state } from './state.js'; +import { query, isAbortError } from './api.js'; +import { getHostFilter, getTable, getTimeBucketStep } from './time.js'; +import { getFacetFilters } from './breakdowns/index.js'; +import { buildLogColumnsSql, LOG_COLUMN_ORDER } from './columns.js'; +import { loadSql } from './sql-loader.js'; +import { + buildLogRowHtml, buildLogTableHeaderHtml, +} from './templates/logs-table.js'; +import { createLimiter } from './concurrency-limiter.js'; + +const SECOND_MS = 1000; +const MINUTE_MS = 60 * SECOND_MS; +const HOUR_MS = 60 * MINUTE_MS; +const DAY_MS = 24 * HOUR_MS; + +/** + * Parse a ClickHouse INTERVAL string to milliseconds. + * @param {string} interval + * @returns {number} + */ +function parseIntervalToMs(interval) { + const match = interval.match(/INTERVAL\s+(\d+)\s+(\w+)/i); + if (!match) return MINUTE_MS; + const amount = parseInt(match[1], 10); + const unit = match[2].toUpperCase().replace(/S$/, ''); + const multipliers = { + SECOND: SECOND_MS, + MINUTE: MINUTE_MS, + HOUR: HOUR_MS, + DAY: DAY_MS, + }; + return amount * (multipliers[unit] || MINUTE_MS); +} + +/** + * Format a Date as 'YYYY-MM-DD HH:MM:SS.mmm' in UTC. + * @param {Date} date + * @returns {string} + */ +function formatTimestampUTC(date) { + const pad = (n) => String(n).padStart(2, '0'); + const ms = String(date.getUTCMilliseconds()).padStart(3, '0'); + return `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())} ${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}.${ms}`; +} + +// Concurrency limiter and abort state +const bucketFetchLimiter = createLimiter(4); +// eslint-disable-next-line prefer-const -- reassigned in setup/teardown +let fetchController = null; +// eslint-disable-next-line prefer-const -- reassigned in setup/teardown +let observer = null; +const loadedBuckets = new Set(); + +/** + * Fetch log rows for a specific bucket time window. + * @param {string} bucketTs - Bucket start timestamp + * @param {number} limit - Max rows to fetch + * @param {number} offset - Row offset within the bucket + * @param {AbortSignal} signal - Abort signal + * @returns {Promise} + */ +async function fetchBucketRows(bucketTs, limit, offset, signal) { + const stepMs = parseIntervalToMs(getTimeBucketStep()); + const start = new Date(`${bucketTs.replace(' ', 'T')}Z`); + const end = new Date(start.getTime() + stepMs); + + const startStr = formatTimestampUTC(start); + const endStr = formatTimestampUTC(end); + + const timeFilter = `timestamp >= toDateTime64('${startStr}', 3)` + + ` AND timestamp < toDateTime64('${endStr}', 3)`; + + const sql = await loadSql('logs', { + database: DATABASE, + table: getTable(), + columns: buildLogColumnsSql(state.pinnedColumns), + timeFilter, + hostFilter: getHostFilter(), + facetFilters: getFacetFilters(), + additionalWhereClause: state.additionalWhereClause, + pageSize: String(limit), + }); + + const finalSql = offset > 0 + ? `${sql.trimEnd().replace(/\n$/, '')} OFFSET ${offset}\n` + : sql; + + const result = await query(finalSql, { signal }); + return result.data; +} + +/** + * Replace a placeholder with actual data rows. + */ +function replacePlaceholder(placeholder, rows, cols, pin, offsets) { + if (!placeholder || !placeholder.parentNode) return; + + let html = ''; + for (let i = 0; i < rows.length; i += 1) { + html += buildLogRowHtml({ + row: rows[i], columns: cols, rowIdx: i, pinned: pin, pinnedOffsets: offsets, + }); + } + + if (html) { + placeholder.insertAdjacentHTML('afterend', html); + } + placeholder.remove(); +} + +/** + * Load data for a single bucket (head and optionally tail). + */ +async function loadBucket(ts, bucket, cols, pin, offsets, signal) { + const headEl = document.getElementById(`bucket-head-${ts}`); + if (headEl) { + try { + const fn = () => fetchBucketRows(ts, bucket.headCount, 0, signal); + const rows = await bucketFetchLimiter(fn); + if (!signal.aborted) { + replacePlaceholder(headEl, rows, cols, pin, offsets); + } + } catch (err) { + if (!isAbortError(err)) { + // eslint-disable-next-line no-console + console.error('Bucket head fetch error:', err); + } + } + } + + const tailEl = document.getElementById(`bucket-tail-${ts}`); + if (tailEl && bucket.tailCount > 0) { + try { + const fn = () => fetchBucketRows(ts, bucket.tailCount, 500, signal); + const rows = await bucketFetchLimiter(fn); + if (!signal.aborted) { + replacePlaceholder(tailEl, rows, cols, pin, offsets); + } + } catch (err) { + if (!isAbortError(err)) { + // eslint-disable-next-line no-console + console.error('Bucket tail fetch error:', err); + } + } + } +} + +/** + * Build the header HTML using real log column definitions. + * @returns {{ headerHtml: string, columns: string[], numColumns: number }} + */ +export function buildBucketHeader() { + const columns = LOG_COLUMN_ORDER; + const pinned = state.pinnedColumns; + const pinnedOffsets = {}; + const headerHtml = buildLogTableHeaderHtml(columns, pinned, pinnedOffsets); + return { + headerHtml, columns, numColumns: columns.length, pinned, pinnedOffsets, + }; +} + +/** + * Set up an IntersectionObserver for lazy bucket data loading. + * @param {HTMLElement} container - Scroll container with bucket rows + * @param {Map} bucketMap - timestamp → bucket metadata + * @param {string[]} columns + * @param {string[]} pinned + * @param {Record} pinnedOffsets + */ +export function setupBucketObserver(container, bucketMap, columns, pinned, pinnedOffsets) { + // Abort previous fetches + if (fetchController) fetchController.abort(); + fetchController = new AbortController(); + if (observer) observer.disconnect(); + loadedBuckets.clear(); + + const { signal } = fetchController; + observer = new IntersectionObserver((entries) => { + for (const entry of entries) { + if (entry.isIntersecting) { + const row = entry.target; + const ts = row.id.replace('bucket-head-', ''); + if (!loadedBuckets.has(ts)) { + loadedBuckets.add(ts); + observer.unobserve(row); + const bucket = bucketMap.get(ts); + if (bucket) { + loadBucket(ts, bucket, columns, pinned, pinnedOffsets, signal); + } + } + } + } + }, { rootMargin: '200px 0px', threshold: 0 }); + + const headRows = container.querySelectorAll('tbody tr.bucket-head'); + for (const row of headRows) { + observer.observe(row); + } +} + +/** + * Clean up observer and abort in-flight bucket fetches. + */ +export function teardownBucketLoader() { + if (fetchController) { + fetchController.abort(); + fetchController = null; + } + if (observer) { + observer.disconnect(); + observer = null; + } + loadedBuckets.clear(); +} diff --git a/js/bucket-table.test.js b/js/bucket-table.test.js index ea5f1c0..881720e 100644 --- a/js/bucket-table.test.js +++ b/js/bucket-table.test.js @@ -11,6 +11,7 @@ */ import { assert } from 'chai'; import { computeBucketHeights, renderBucketTable } from './logs.js'; +import { LOG_COLUMN_ORDER } from './columns.js'; function makeChartData(count, baseCnt = 10) { return Array.from({ length: count }, (_, i) => ({ @@ -332,4 +333,35 @@ describe('renderBucketTable', () => { const tailH = parseInt(tailRow.style.height, 10); assert.strictEqual(headH + tailH, 800 * 28); }); + + it('renders actual column headers instead of generic "Log Buckets"', () => { + const data = makeChartData(2); + renderBucketTable(container, data); + const thElements = container.querySelectorAll('thead th'); + assert.strictEqual(thElements.length, LOG_COLUMN_ORDER.length); + // Should not contain old generic header + const headerText = container.querySelector('thead').textContent; + assert.notInclude(headerText, 'Log Buckets'); + }); + + it('placeholder colspan matches column count', () => { + const data = [ + { + t: '2026-01-15 00:00:00.000', cnt_ok: '10', cnt_4xx: '0', cnt_5xx: '0', + }, + ]; + renderBucketTable(container, data); + const td = container.querySelector('.bucket-placeholder'); + assert.strictEqual(td.getAttribute('colspan'), String(LOG_COLUMN_ORDER.length)); + }); + + it('column headers have data-action for pin toggling', () => { + const data = makeChartData(1); + renderBucketTable(container, data); + const thElements = container.querySelectorAll('thead th'); + for (const th of thElements) { + assert.strictEqual(th.dataset.action, 'toggle-pinned-column'); + assert.ok(th.dataset.col, 'th should have data-col attribute'); + } + }); }); diff --git a/js/logs.js b/js/logs.js index bf57852..d47e7c2 100644 --- a/js/logs.js +++ b/js/logs.js @@ -21,10 +21,12 @@ import { formatBytes } from './format.js'; import { getColorForColumn } from './colors/index.js'; import { getRequestContext, isRequestCurrent } from './request-context.js'; import { - LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, LOG_COLUMN_TO_FACET, buildLogColumnsSql, + LOG_COLUMN_ORDER, LOG_COLUMN_SHORT_LABELS, buildLogColumnsSql, } from './columns.js'; import { loadSql } from './sql-loader.js'; -import { formatLogCell } from './templates/logs-table.js'; +import { + buildBucketHeader, setupBucketObserver, teardownBucketLoader, +} from './bucket-loader.js'; import { PAGE_SIZE, INITIAL_PAGE_SIZE } from './pagination.js'; import { setScrubberPosition } from './chart.js'; import { parseUTC } from './chart-state.js'; @@ -403,29 +405,6 @@ export function initChartCollapseToggle() { updateCollapseToggleLabel(); } -/** - * renderCell callback for VirtualTable (unused while bucket-row table is active). - * Returns HTML string for a single cell. - */ -// eslint-disable-next-line no-unused-vars -- kept for future VirtualTable re-enablement -function renderCell(col, value) { - const { displayValue, cellClass, colorIndicator } = formatLogCell(col.key, value); - const escaped = escapeHtml(displayValue); - - // Add click-to-filter attributes when column has a facet mapping and a color indicator - let actionAttrs = ''; - let extraClass = ''; - const facetMapping = LOG_COLUMN_TO_FACET[col.key]; - if (colorIndicator && facetMapping && value !== null && value !== undefined && value !== '') { - const filterValue = facetMapping.transform ? facetMapping.transform(value) : String(value); - actionAttrs = ` data-action="add-filter" data-col="${escapeHtml(facetMapping.col)}" data-value="${escapeHtml(filterValue)}" data-exclude="false"`; - extraClass = ' clickable'; - } - - const cls = cellClass || extraClass ? ` class="${(cellClass || '') + extraClass}"` : ''; - return `${colorIndicator}${escaped}`; -} - /** * Find the nearest cached cursor for a given page index. * @param {number} pageIdx @@ -702,8 +681,17 @@ export function renderBucketTable(el, chartData) { const { buckets } = computeBucketHeights(chartData); - // Build table HTML - const numColumns = 7; // placeholder colspan + // Build column list and header from bucket-loader + const { + headerHtml, columns, numColumns, pinned, pinnedOffsets, + } = buildBucketHeader(); + + // Build a lookup from timestamp to bucket metadata + const bucketMap = new Map(); + for (const b of buckets) { + bucketMap.set(b.t, b); + } + let tbodyHtml = ''; // Reverse to newest-first (chart data is oldest-first) @@ -715,24 +703,32 @@ export function renderBucketTable(el, chartData) { if (b.tailCount > 0) { headLabel = `500 of ${b.count.toLocaleString()} rows`; } else { - headLabel = b.count === 1 ? '1 row' : `${b.count.toLocaleString()} rows`; + headLabel = b.count === 1 + ? '1 row' + : `${b.count.toLocaleString()} rows`; } - tbodyHtml += `` - + `${headLabel}` + tbodyHtml += '` + + `${headLabel}` + ''; // Tail row (only when bucket has > 500 rows) if (b.tailCount > 0) { const tailLabel = `${b.tailCount.toLocaleString()} remaining rows`; - tbodyHtml += `` - + `${tailLabel}` + tbodyHtml += '` + + `${tailLabel}` + ''; } } // eslint-disable-next-line no-param-reassign -- DOM manipulation el.innerHTML = ` - + ${headerHtml}${tbodyHtml}
Log Buckets
`; @@ -746,12 +742,16 @@ export function renderBucketTable(el, chartData) { syncBucketScrubber(el); }; el.addEventListener('scroll', bucketScrollHandler, { passive: true }); + + // Set up IntersectionObserver for lazy bucket data loading + setupBucketObserver(el, bucketMap, columns, pinned, pinnedOffsets); } /** - * Clean up bucket table event listeners. + * Clean up bucket table event listeners, observer, and in-flight fetches. */ function destroyBucketTable() { + teardownBucketLoader(); if (bucketTableContainer && bucketScrollHandler) { bucketTableContainer.removeEventListener('scroll', bucketScrollHandler); } From 8c0f2508a660343f3ff7877bc7e77c4d3fde5872 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 13 Feb 2026 20:22:26 +0100 Subject: [PATCH 28/28] feat: DOM virtualization with 2000-row cap for bucket logs Add eviction logic to bucket-loader.js so off-screen buckets get collapsed back to placeholders, keeping the DOM under 2000 data rows. Also abort in-flight HTTP requests for buckets that scroll out of view (not just on navigation change). - Per-bucket AbortControllers cancel fetches on scroll-past - Sentinel rows + data-bucket attributes for fast row grouping - Eviction observer (800px margin) proactively evicts off-screen data - enforceRowBudget() evicts farthest bucket when over 2000 rows - LRU head cache (20 buckets) avoids re-fetching on scroll-back - Budget-aware tail loading skips if DOM is near capacity - 17 new tests, 841 total pass, lint clean Co-Authored-By: Claude Opus 4.6 Signed-off-by: Lars Trieloff --- js/bucket-loader.js | 266 ++++++++++++++++++++++++++++++--- js/bucket-loader.test.js | 313 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 557 insertions(+), 22 deletions(-) create mode 100644 js/bucket-loader.test.js diff --git a/js/bucket-loader.js b/js/bucket-loader.js index 56c53b1..bb6f415 100644 --- a/js/bucket-loader.js +++ b/js/bucket-loader.js @@ -13,7 +13,8 @@ /** * Bucket data loader — extracted from logs.js to stay within the * max-lines lint limit. Handles per-bucket on-demand data fetching, - * IntersectionObserver setup, and placeholder replacement. + * IntersectionObserver setup, placeholder replacement, and DOM + * virtualization with a 2000-row cap. */ import { DATABASE } from './config.js'; @@ -33,6 +34,10 @@ const MINUTE_MS = 60 * SECOND_MS; const HOUR_MS = 60 * MINUTE_MS; const DAY_MS = 24 * HOUR_MS; +const MAX_DOM_ROWS = 2000; +const HEAD_CACHE_SIZE = 20; +const ROW_HEIGHT = 28; + /** * Parse a ClickHouse INTERVAL string to milliseconds. * @param {string} interval @@ -68,9 +73,33 @@ const bucketFetchLimiter = createLimiter(4); // eslint-disable-next-line prefer-const -- reassigned in setup/teardown let fetchController = null; // eslint-disable-next-line prefer-const -- reassigned in setup/teardown -let observer = null; +let loadObserver = null; +// eslint-disable-next-line prefer-const -- reassigned in setup/teardown +let evictionObserver = null; const loadedBuckets = new Set(); +// Per-bucket AbortControllers for cancelling in-flight fetches +const bucketControllers = new Map(); + +// LRU head-data cache (ts → rows) +const headCache = new Map(); + +// Stored container reference for eviction observer sentinel wiring +// eslint-disable-next-line prefer-const -- reassigned in setup/teardown +let storedContainer = null; + +/** + * Store rows in the LRU head cache. + */ +function cacheHead(ts, rows) { + headCache.delete(ts); + headCache.set(ts, rows); + if (headCache.size > HEAD_CACHE_SIZE) { + const oldest = headCache.keys().next().value; + headCache.delete(oldest); + } +} + /** * Fetch log rows for a specific bucket time window. * @param {string} bucketTs - Bucket start timestamp @@ -110,35 +139,168 @@ async function fetchBucketRows(bucketTs, limit, offset, signal) { } /** - * Replace a placeholder with actual data rows. + * Count data rows currently in the DOM. + */ +function countDataRows(container) { + return container.querySelectorAll('tr[data-bucket]').length; +} + +/** + * Evict a loaded bucket back to a placeholder. + */ +function evictBucket(ts, container, cols) { + // Abort any in-flight fetch + const ctrl = bucketControllers.get(ts); + if (ctrl) { + ctrl.abort(); + bucketControllers.delete(ts); + } + + // Find sentinel + const sentinel = container.querySelector(`tr.bucket-sentinel[data-bucket="${ts}"]`); + if (!sentinel) return; + + // Stop observing sentinel before removal + if (evictionObserver) evictionObserver.unobserve(sentinel); + + // Collect data rows following the sentinel + const rows = []; + let next = sentinel.nextElementSibling; + while (next && next.dataset.bucket === ts) { + rows.push(next); + next = next.nextElementSibling; + } + + // Calculate replacement height + const totalHeight = rows.length * ROW_HEIGHT; + const numColumns = cols.length; + + // Create placeholder + const placeholder = document.createElement('tr'); + placeholder.id = `bucket-head-${ts}`; + placeholder.className = 'bucket-row bucket-head'; + placeholder.style.height = `${totalHeight}px`; + // eslint-disable-next-line no-irregular-whitespace + placeholder.innerHTML = `${rows.length.toLocaleString()} rows (evicted)`; + + // Replace sentinel with placeholder, remove data rows + sentinel.parentNode.insertBefore(placeholder, sentinel); + sentinel.remove(); + for (const r of rows) r.remove(); + + // Remove from loaded set + loadedBuckets.delete(ts); + + // Re-observe with load observer + if (loadObserver) loadObserver.observe(placeholder); +} + +/** + * Enforce the MAX_DOM_ROWS budget by evicting the farthest bucket. + */ +function enforceRowBudget(container, cols) { + const viewportCenter = window.innerHeight / 2; + + // eslint-disable-next-line no-constant-condition + while (true) { + const currentCount = countDataRows(container); + if (currentCount <= MAX_DOM_ROWS) break; + + // Find farthest loaded bucket from viewport center + let farthestTs = null; + let farthestDist = -1; + + for (const ts of loadedBuckets) { + const sentinel = container.querySelector( + `tr.bucket-sentinel[data-bucket="${ts}"]`, + ); + if (sentinel) { + const rect = sentinel.getBoundingClientRect(); + const dist = Math.abs(rect.top - viewportCenter); + if (dist > farthestDist) { + farthestDist = dist; + farthestTs = ts; + } + } + } + + if (!farthestTs) break; + evictBucket(farthestTs, container, cols); + } +} + +/** + * Replace a placeholder with a sentinel and actual data rows. */ -function replacePlaceholder(placeholder, rows, cols, pin, offsets) { +function replacePlaceholder(placeholder, rows, cols, pin, offsets, ts) { if (!placeholder || !placeholder.parentNode) return; - let html = ''; + // Build sentinel + data rows HTML + let html = `'; + for (let i = 0; i < rows.length; i += 1) { - html += buildLogRowHtml({ + const rowHtml = buildLogRowHtml({ row: rows[i], columns: cols, rowIdx: i, pinned: pin, pinnedOffsets: offsets, }); + // Inject data-bucket attribute into the tag + html += rowHtml.replace(' controller.abort(), { once: true }); + } + } + + return controller.signal; } /** * Load data for a single bucket (head and optionally tail). */ -async function loadBucket(ts, bucket, cols, pin, offsets, signal) { +async function loadBucket(ts, bucket, cols, pin, offsets, globalSignal, container) { + const signal = createBucketSignal(ts, globalSignal); + const headEl = document.getElementById(`bucket-head-${ts}`); if (headEl) { try { - const fn = () => fetchBucketRows(ts, bucket.headCount, 0, signal); - const rows = await bucketFetchLimiter(fn); + let rows; + if (headCache.has(ts)) { + rows = headCache.get(ts); + } else { + const fn = () => fetchBucketRows(ts, bucket.headCount, 0, signal); + rows = await bucketFetchLimiter(fn); + if (!signal.aborted) cacheHead(ts, rows); + } if (!signal.aborted) { - replacePlaceholder(headEl, rows, cols, pin, offsets); + replacePlaceholder(headEl, rows, cols, pin, offsets, ts); + enforceRowBudget(container, cols); } } catch (err) { if (!isAbortError(err)) { @@ -150,11 +312,18 @@ async function loadBucket(ts, bucket, cols, pin, offsets, signal) { const tailEl = document.getElementById(`bucket-tail-${ts}`); if (tailEl && bucket.tailCount > 0) { + // Budget-awareness: check remaining room before tail fetch + const currentRows = countDataRows(container); + const budget = MAX_DOM_ROWS - currentRows; + if (budget <= 0) return; + const effectiveTailLimit = Math.min(bucket.tailCount, budget); + try { - const fn = () => fetchBucketRows(ts, bucket.tailCount, 500, signal); + const fn = () => fetchBucketRows(ts, effectiveTailLimit, 500, signal); const rows = await bucketFetchLimiter(fn); if (!signal.aborted) { - replacePlaceholder(tailEl, rows, cols, pin, offsets); + replacePlaceholder(tailEl, rows, cols, pin, offsets, ts); + enforceRowBudget(container, cols); } } catch (err) { if (!isAbortError(err)) { @@ -163,6 +332,9 @@ async function loadBucket(ts, bucket, cols, pin, offsets, signal) { } } } + + // Clean up per-bucket controller after completion + bucketControllers.delete(ts); } /** @@ -180,7 +352,7 @@ export function buildBucketHeader() { } /** - * Set up an IntersectionObserver for lazy bucket data loading. + * Set up IntersectionObservers for lazy bucket data loading and eviction. * @param {HTMLElement} container - Scroll container with bucket rows * @param {Map} bucketMap - timestamp → bucket metadata * @param {string[]} columns @@ -191,44 +363,94 @@ export function setupBucketObserver(container, bucketMap, columns, pinned, pinne // Abort previous fetches if (fetchController) fetchController.abort(); fetchController = new AbortController(); - if (observer) observer.disconnect(); + if (loadObserver) loadObserver.disconnect(); + if (evictionObserver) evictionObserver.disconnect(); loadedBuckets.clear(); + bucketControllers.forEach((c) => c.abort()); + bucketControllers.clear(); + headCache.clear(); + + // Store container reference for eviction observer sentinel wiring + storedContainer = container; const { signal } = fetchController; - observer = new IntersectionObserver((entries) => { + + // Load observer: triggers data fetch when placeholder enters viewport + loadObserver = new IntersectionObserver((entries) => { for (const entry of entries) { if (entry.isIntersecting) { const row = entry.target; const ts = row.id.replace('bucket-head-', ''); if (!loadedBuckets.has(ts)) { loadedBuckets.add(ts); - observer.unobserve(row); + loadObserver.unobserve(row); const bucket = bucketMap.get(ts); if (bucket) { - loadBucket(ts, bucket, columns, pinned, pinnedOffsets, signal); + loadBucket(ts, bucket, columns, pinned, pinnedOffsets, signal, container); } } } } }, { rootMargin: '200px 0px', threshold: 0 }); + // Eviction observer: evicts buckets that scroll far out of view + evictionObserver = new IntersectionObserver((entries) => { + for (const entry of entries) { + if (!entry.isIntersecting) { + const sentinel = entry.target; + const ts = sentinel.dataset.bucket; + if (ts && loadedBuckets.has(ts)) { + evictBucket(ts, container, columns); + } + } + } + }, { rootMargin: '800px 0px', threshold: 0 }); + const headRows = container.querySelectorAll('tbody tr.bucket-head'); for (const row of headRows) { - observer.observe(row); + loadObserver.observe(row); } } /** - * Clean up observer and abort in-flight bucket fetches. + * Clean up observers and abort in-flight bucket fetches. */ export function teardownBucketLoader() { + // Abort all per-bucket controllers + bucketControllers.forEach((c) => c.abort()); + bucketControllers.clear(); + + // Abort global controller if (fetchController) { fetchController.abort(); fetchController = null; } - if (observer) { - observer.disconnect(); - observer = null; + + // Disconnect both observers + if (loadObserver) { + loadObserver.disconnect(); + loadObserver = null; + } + if (evictionObserver) { + evictionObserver.disconnect(); + evictionObserver = null; } + loadedBuckets.clear(); + headCache.clear(); + + // Clear stored container reference + storedContainer = null; } + +// Exported for testing only +export { + evictBucket as _evictBucket, + enforceRowBudget as _enforceRowBudget, + countDataRows as _countDataRows, + replacePlaceholder as _replacePlaceholder, + cacheHead as _cacheHead, + headCache as _headCache, + loadedBuckets as _loadedBuckets, + MAX_DOM_ROWS as _MAX_DOM_ROWS, +}; diff --git a/js/bucket-loader.test.js b/js/bucket-loader.test.js new file mode 100644 index 0000000..dc36ba9 --- /dev/null +++ b/js/bucket-loader.test.js @@ -0,0 +1,313 @@ +/* + * Copyright 2026 Adobe. All rights reserved. + * This file is licensed to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. You may obtain a copy + * of the License at https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under + * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS + * OF ANY KIND, either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +import { assert } from 'chai'; +import { + teardownBucketLoader, + _evictBucket, + _enforceRowBudget, + _countDataRows, + _replacePlaceholder, + _cacheHead, + _headCache, + _loadedBuckets, + _MAX_DOM_ROWS, +} from './bucket-loader.js'; +import { LOG_COLUMN_ORDER } from './columns.js'; + +const COLS = LOG_COLUMN_ORDER; +const TS = '2026-01-15 00:00:00.000'; +const TS2 = '2026-01-15 00:01:00.000'; +const TS3 = '2026-01-15 00:02:00.000'; + +/** + * Build a minimal fake row matching the column order. + */ +function fakeRow(idx) { + const row = {}; + for (const col of COLS) { + if (col === 'timestamp') row[col] = `2026-01-15T00:00:0${idx}.000Z`; + else if (col === 'response.status') row[col] = 200; + else if (col === 'response.body_size') row[col] = 1024; + else if (col === 'request.method') row[col] = 'GET'; + else row[col] = `val-${idx}`; + } + return row; +} + +/** + * Create a container with a containing a bucket-head placeholder. + */ +function createContainer(ts, numRows) { + const container = document.createElement('div'); + const table = document.createElement('table'); + const tbody = document.createElement('tbody'); + const tr = document.createElement('tr'); + tr.id = `bucket-head-${ts}`; + tr.className = 'bucket-row bucket-head'; + tr.style.height = `${numRows * 28}px`; + tr.innerHTML = `${numRows} rows`; + tbody.appendChild(tr); + table.appendChild(tbody); + container.appendChild(table); + document.body.appendChild(container); + return container; +} + +/** + * Manually insert sentinel + data rows to simulate a loaded bucket. + */ +function insertLoadedBucket(container, ts, rowCount) { + const tbody = container.querySelector('tbody'); + // Sentinel + const sentinel = document.createElement('tr'); + sentinel.className = 'bucket-sentinel'; + sentinel.dataset.bucket = ts; + sentinel.style.cssText = 'height:0;padding:0;border:0;line-height:0;visibility:hidden;'; + tbody.appendChild(sentinel); + // Data rows + for (let i = 0; i < rowCount; i += 1) { + const tr = document.createElement('tr'); + tr.dataset.bucket = ts; + tr.dataset.rowIdx = String(i); + tr.innerHTML = `row ${i}`; + tbody.appendChild(tr); + } + _loadedBuckets.add(ts); +} + +describe('bucket-loader virtualization', () => { + let container; + + afterEach(() => { + teardownBucketLoader(); + if (container && container.parentNode) { + container.parentNode.removeChild(container); + } + container = null; + }); + + describe('countDataRows', () => { + it('counts rows with data-bucket attribute', () => { + container = createContainer(TS, 10); + insertLoadedBucket(container, TS, 5); + const count = _countDataRows(container); + // 5 data rows + 1 sentinel = 6 tr[data-bucket] + assert.strictEqual(count, 6); + }); + + it('returns 0 for empty container', () => { + container = createContainer(TS, 0); + assert.strictEqual(_countDataRows(container), 0); + }); + }); + + describe('replacePlaceholder', () => { + it('inserts sentinel before data rows', () => { + container = createContainer(TS, 3); + const placeholder = document.getElementById(`bucket-head-${TS}`); + const rows = [fakeRow(0), fakeRow(1), fakeRow(2)]; + + _replacePlaceholder(placeholder, rows, COLS, [], {}, TS); + + const sentinel = container.querySelector('tr.bucket-sentinel'); + assert.ok(sentinel, 'sentinel should exist'); + assert.strictEqual(sentinel.dataset.bucket, TS); + }); + + it('tags each data row with data-bucket', () => { + container = createContainer(TS, 2); + const placeholder = document.getElementById(`bucket-head-${TS}`); + const rows = [fakeRow(0), fakeRow(1)]; + + _replacePlaceholder(placeholder, rows, COLS, [], {}, TS); + + const dataRows = container.querySelectorAll('tr[data-bucket]'); + // 1 sentinel + 2 data rows + assert.strictEqual(dataRows.length, 3); + }); + + it('removes the placeholder element', () => { + container = createContainer(TS, 1); + const placeholder = document.getElementById(`bucket-head-${TS}`); + const rows = [fakeRow(0)]; + + _replacePlaceholder(placeholder, rows, COLS, [], {}, TS); + + assert.isNull(document.getElementById(`bucket-head-${TS}`)); + }); + + it('handles null placeholder gracefully', () => { + container = createContainer(TS, 0); + // Should not throw + _replacePlaceholder(null, [fakeRow(0)], COLS, [], {}, TS); + }); + + it('handles empty rows', () => { + container = createContainer(TS, 0); + const placeholder = document.getElementById(`bucket-head-${TS}`); + + _replacePlaceholder(placeholder, [], COLS, [], {}, TS); + + // Sentinel should still be inserted + const sentinel = container.querySelector('tr.bucket-sentinel'); + assert.ok(sentinel, 'sentinel should exist even with no data rows'); + }); + }); + + describe('evictBucket', () => { + it('replaces sentinel + data rows with a placeholder', () => { + container = createContainer(TS, 10); + // Remove the initial placeholder first + const initial = document.getElementById(`bucket-head-${TS}`); + if (initial) initial.remove(); + + insertLoadedBucket(container, TS, 5); + + _evictBucket(TS, container, COLS); + + // Sentinel should be gone + assert.isNull(container.querySelector('tr.bucket-sentinel')); + // Data rows should be gone + assert.strictEqual(container.querySelectorAll('tr[data-bucket]').length, 0); + // New placeholder should exist + const placeholder = document.getElementById(`bucket-head-${TS}`); + assert.ok(placeholder, 'placeholder should be recreated'); + assert.include(placeholder.className, 'bucket-head'); + assert.include(placeholder.textContent, 'evicted'); + }); + + it('removes ts from loadedBuckets', () => { + container = createContainer(TS, 5); + const initial = document.getElementById(`bucket-head-${TS}`); + if (initial) initial.remove(); + + insertLoadedBucket(container, TS, 3); + assert.isTrue(_loadedBuckets.has(TS)); + + _evictBucket(TS, container, COLS); + assert.isFalse(_loadedBuckets.has(TS)); + }); + + it('does nothing if sentinel not found', () => { + container = createContainer(TS, 5); + // No sentinel inserted, just placeholder + _evictBucket(TS, container, COLS); + // Should not crash, placeholder should still be there + assert.ok(document.getElementById(`bucket-head-${TS}`)); + }); + + it('placeholder height matches evicted row count', () => { + container = createContainer(TS, 10); + const initial = document.getElementById(`bucket-head-${TS}`); + if (initial) initial.remove(); + + insertLoadedBucket(container, TS, 7); + + _evictBucket(TS, container, COLS); + + const placeholder = document.getElementById(`bucket-head-${TS}`); + assert.strictEqual(placeholder.style.height, `${7 * 28}px`); + }); + }); + + describe('enforceRowBudget', () => { + it('does not evict when under budget', () => { + container = createContainer(TS, 10); + const initial = document.getElementById(`bucket-head-${TS}`); + if (initial) initial.remove(); + + insertLoadedBucket(container, TS, 5); + + _enforceRowBudget(container, COLS); + + // Should still be loaded + assert.isTrue(_loadedBuckets.has(TS)); + }); + + it('evicts farthest bucket when over budget', () => { + container = document.createElement('div'); + const table = document.createElement('table'); + const tbody = document.createElement('tbody'); + table.appendChild(tbody); + container.appendChild(table); + document.body.appendChild(container); + + // Insert three loaded buckets, each with MAX_DOM_ROWS/2 rows + const halfBudget = Math.ceil(_MAX_DOM_ROWS / 2); + insertLoadedBucket(container, TS, halfBudget); + insertLoadedBucket(container, TS2, halfBudget); + insertLoadedBucket(container, TS3, halfBudget); + + _enforceRowBudget(container, COLS); + + // At least one bucket should have been evicted + const remaining = _loadedBuckets.size; + assert.isBelow(remaining, 3, 'at least one bucket should be evicted'); + }); + }); + + describe('headCache (LRU)', () => { + afterEach(() => { + _headCache.clear(); + }); + + it('stores and retrieves rows', () => { + const rows = [fakeRow(0)]; + _cacheHead(TS, rows); + assert.strictEqual(_headCache.get(TS), rows); + }); + + it('evicts oldest entry when over capacity', () => { + // Fill cache to capacity (20) + 1 + for (let i = 0; i < 21; i += 1) { + const ts = `2026-01-15 00:${String(i).padStart(2, '0')}:00.000`; + _cacheHead(ts, [fakeRow(i)]); + } + assert.strictEqual(_headCache.size, 20); + // First entry should be evicted + assert.isFalse(_headCache.has('2026-01-15 00:00:00.000')); + // Last entry should exist + assert.isTrue(_headCache.has('2026-01-15 00:20:00.000')); + }); + + it('re-accessing entry moves it to end (LRU refresh)', () => { + _cacheHead(TS, [fakeRow(0)]); + _cacheHead(TS2, [fakeRow(1)]); + // Re-cache TS to move it to end + _cacheHead(TS, [fakeRow(0)]); + + const keys = Array.from(_headCache.keys()); + assert.strictEqual(keys[keys.length - 1], TS, 'TS should be at end after re-access'); + }); + }); + + describe('teardownBucketLoader', () => { + it('clears loadedBuckets', () => { + _loadedBuckets.add(TS); + _loadedBuckets.add(TS2); + teardownBucketLoader(); + assert.strictEqual(_loadedBuckets.size, 0); + }); + + it('clears headCache', () => { + _cacheHead(TS, [fakeRow(0)]); + teardownBucketLoader(); + assert.strictEqual(_headCache.size, 0); + }); + }); + + describe('MAX_DOM_ROWS constant', () => { + it('is set to 2000', () => { + assert.strictEqual(_MAX_DOM_ROWS, 2000); + }); + }); +});