move overscan to RowsRangeProvider and make row ranges more explicit

src/components/HighTable/Scroller.tsx

@@ -5,21 +5,16 @@ import { CellNavigationContext } from '../../contexts/CellNavigationContext.js'
 import { DataContext } from '../../contexts/DataContext.js'
 import { RowsAndColumnsContext } from '../../contexts/RowsAndColumnsContext.js'
 import styles from '../../HighTable.module.css'
-import { ariaOffset, defaultOverscan, rowHeight } from './constants.js'
+import { ariaOffset, rowHeight } from './constants.js'
 
-export interface ScrollerProps {
-  overscan?: number // number of rows to fetch outside of the viewport
-}
-
-type Props = {
+interface Props {
   headerHeight?: number // height of the table header
   setViewportWidth: (width: number) => void // callback to set the current viewport width
   children?: React.ReactNode
-} & ScrollerProps
+}
 
 export default function Scroller({
   headerHeight = rowHeight,
-  overscan = defaultOverscan,
   setViewportWidth,
   children,
 }: Props) {
@@ -32,7 +27,7 @@ export default function Scroller({
   const { numRows } = useContext(DataContext)
   const { onScrollKeyDown } = useContext(CellNavigationContext)
   const { shouldScroll, setShouldScroll, cellPosition } = useContext(CellNavigationContext)
-  const { rowsRangeWithPadding, setRowsRange } = useContext(RowsAndColumnsContext)
+  const { fetchedRowsRange, renderedRowsRange, setVisibleRowsRange } = useContext(RowsAndColumnsContext)
 
   /**
    * Compute the values:
@@ -55,19 +50,15 @@ export default function Scroller({
     // TODO(SL): remove this fallback? It's only for the tests, where the elements have zero height
     const clientHeight = viewportHeight === 0 ? 100 : viewportHeight
 
-    // determine rows to fetch based on current scroll position (indexes refer to the virtual table domain)
-    const startView = Math.floor(numRows * scrollTop / scrollHeight)
-    const endView = Math.ceil(numRows * (scrollTop + clientHeight) / scrollHeight)
-    const start = Math.max(0, startView - overscan)
-    const end = Math.min(numRows, endView + overscan)
+    // determine visible rows based on current scroll position (indexes refer to the virtual table domain)
+    const start = Math.max(0, Math.floor(numRows * scrollTop / scrollHeight))
+    const end = Math.min(numRows, Math.ceil(numRows * (scrollTop + clientHeight) / scrollHeight))
 
     if (isNaN(start)) throw new Error(`invalid start row ${start}`)
     if (isNaN(end)) throw new Error(`invalid end row ${end}`)
     if (end - start > 1000) throw new Error(`attempted to render too many rows ${end - start} table must be contained in a scrollable div`)
-
-    const rowsRange = { start, end }
-    setRowsRange?.(rowsRange)
-  }, [numRows, overscan, scrollHeight, setRowsRange])
+    setVisibleRowsRange?.({ start, end })
+  }, [numRows, scrollHeight, setVisibleRowsRange])
 
   /**
    * Handle keyboard events for scrolling
@@ -89,22 +80,23 @@ export default function Scroller({
    * back to it
    */
   useEffect(() => {
-    if (!shouldScroll || scrollTop === undefined || scrollToTop === undefined || rowsRangeWithPadding === undefined) {
+    if (!shouldScroll || scrollTop === undefined || scrollToTop === undefined || fetchedRowsRange === undefined) {
       return
     }
     setShouldScroll?.(false)
     const row = cellPosition.rowIndex - ariaOffset
     let nextScrollTop = scrollTop
-    // if row outside of the rows range, scroll to the estimated position of the cell,
+    // if the row is outside of the fetched rows range, scroll to the estimated position of the cell,
     // to wait for the cell to be fetched and rendered
-    if (row < rowsRangeWithPadding.start || row >= rowsRangeWithPadding.end) {
+    // TODO(SL): should fetchedRowsRange be replaced with visibleRowsRange?
+    if (row < fetchedRowsRange.start || row >= fetchedRowsRange.end) {
       nextScrollTop = row * rowHeight
     }
     if (nextScrollTop !== scrollTop) {
       // scroll to the cell
       scrollToTop(nextScrollTop)
     }
-  }, [cellPosition, shouldScroll, rowsRangeWithPadding, setShouldScroll, scrollToTop, scrollTop])
+  }, [cellPosition, shouldScroll, fetchedRowsRange, setShouldScroll, scrollToTop, scrollTop])
 
   /**
    * Track viewport size and scroll position
@@ -169,8 +161,8 @@ export default function Scroller({
 
   // Note: it does not depend on headerHeight, because the header is always present in the DOM
   const top = useMemo(() => {
-    return (rowsRangeWithPadding?.startPadding ?? 0) * rowHeight
-  }, [rowsRangeWithPadding])
+    return (renderedRowsRange?.start ?? 0) * rowHeight
+  }, [renderedRowsRange])
 
   return (
     <div className={styles.tableScroll} ref={viewportRef} role="group" aria-labelledby="caption" onKeyDown={onKeyDown} tabIndex={0}>

src/components/HighTable/Slice.tsx

@@ -43,7 +43,7 @@ export default function Slice({
   const { onTableKeyDown: onNavigationTableKeyDown, focusFirstCell } = useContext(CellNavigationContext)
   const { orderBy, onOrderByChange } = useContext(OrderByContext)
   const { selectable, toggleAllRows, pendingSelectionGesture, onTableKeyDown: onSelectionTableKeyDown, allRowsSelected, isRowSelected, toggleRowNumber, toggleRangeToRowNumber } = useContext(SelectionContext)
-  const { columnsParameters, rowsRangeWithPadding } = useContext(RowsAndColumnsContext)
+  const { columnsParameters, renderedRowsRange, fetchedRowsRange } = useContext(RowsAndColumnsContext)
 
   const onTableKeyDown = useCallback((event: KeyboardEvent) => {
     onNavigationTableKeyDown?.(event, { numRowsPerPage })
@@ -76,7 +76,7 @@ export default function Slice({
   // Prepare the slice of data to render
   // TODO(SL): also compute progress percentage here, to show a loading indicator
   const slice = useMemo(() => {
-    if (!rowsRangeWithPadding) {
+    if (!renderedRowsRange || !fetchedRowsRange) {
       return {
         prePadding: [],
         postPadding: [],
@@ -85,11 +85,13 @@ export default function Slice({
         version,
       }
     }
-    const { startPadding, start, end, endPadding } = rowsRangeWithPadding
+    const prePaddingRowCount = Math.max(0, fetchedRowsRange.start - renderedRowsRange.start)
+    const fetchedRowCount = fetchedRowsRange.end - fetchedRowsRange.start
+    const postPaddingRowCount = Math.max(0, renderedRowsRange.end - fetchedRowsRange.end)
     // add empty pre and post rows to fill the viewport
-    const prePadding = Array.from({ length: start - startPadding }, (_, i) => ({ row: startPadding + i }))
-    const rows = Array.from({ length: end - start }, (_, i) => start + i)
-    const postPadding = Array.from({ length: endPadding - end }, (_, i) => ({ row: end + i }))
+    const prePadding = Array.from({ length: prePaddingRowCount }, (_, i) => ({ row: renderedRowsRange.start + i }))
+    const rows = Array.from({ length: fetchedRowCount }, (_, i) => fetchedRowsRange.start + i)
+    const postPadding = Array.from({ length: postPaddingRowCount }, (_, i) => ({ row: fetchedRowsRange.end + i }))
 
     const canMeasureColumn: Record<string, boolean> = {}
     const rowContents = rows.map((row) => {
@@ -112,7 +114,7 @@ export default function Slice({
       canMeasureColumn,
       version,
     }
-  }, [data, columnsParameters, rowsRangeWithPadding, orderBy, version])
+  }, [data, columnsParameters, renderedRowsRange, fetchedRowsRange, orderBy, version])
 
   // don't render table if header is empty
   if (!columnsParameters) return

src/components/HighTable/Wrapper.tsx

@@ -18,7 +18,6 @@ import { RowsAndColumnsProvider } from '../../providers/RowsAndColumnsProvider.j
 import { SelectionProvider } from '../../providers/SelectionProvider.js'
 import { rowHeight } from './constants.js'
 import { columnVisibilityStatesSuffix, columnWidthsSuffix } from './constants.js'
-import type { ScrollerProps } from './Scroller.js'
 import Scroller from './Scroller.js'
 import type { SliceProps } from './Slice.js'
 import Slice from './Slice.js'
@@ -33,7 +32,7 @@ export type WrapperProps = {
   onColumnsVisibilityChange?: (columns: Record<string, MaybeHiddenColumn>) => void // callback which is called whenever the set of hidden columns changes.
   onOrderByChange?: (orderBy: OrderBy) => void // callback to call when a user interaction changes the order. The interactions are disabled if undefined.
   onSelectionChange?: (selection: Selection) => void // callback to call when a user interaction changes the selection. The selection is expressed as data indexes (not as indexes in the table). The interactions are disabled if undefined.
-} & ScrollerProps & RowsAndColumnsProviderProps & SliceProps
+} & RowsAndColumnsProviderProps & SliceProps
 
 export default function Wrapper({
   columnConfiguration,
@@ -100,9 +99,9 @@ export default function Wrapper({
                 <SelectionProvider key={key} selection={selection} onSelectionChange={onSelectionChange} data={data} numRows={numRows}>
                   {/* Create a new navigation context if the dataframe has changed, because the focused cell might not exist anymore */}
                   <CellNavigationProvider key={key}>
-                    <RowsAndColumnsProvider key={key} padding={padding}>
+                    <RowsAndColumnsProvider key={key} padding={padding} overscan={overscan}>
 
-                      <Scroller setViewportWidth={setViewportWidth} overscan={overscan} headerHeight={headerHeight}>
+                      <Scroller setViewportWidth={setViewportWidth} headerHeight={headerHeight}>
                         <Slice
                           setTableCornerSize={setTableCornerSize}
                           {...rest}

src/contexts/RowsAndColumnsContext.ts

@@ -3,19 +3,16 @@ import { createContext } from 'react'
 import type { ColumnParameters } from '../contexts/ColumnParametersContext.js'
 
 export interface RowsRange {
-  start: number
-  end: number
-}
-
-interface RowsRangeWithPadding extends RowsRange {
-  startPadding: number // starting row index (inclusive) of the padded region before `start` (i.e. start - padding rows, or 0)
-  endPadding: number // ending row index (exclusive) of the padded region (i.e. end + padding rows, or numRows)
+  start: number // first row index (inclusive). Indexes refer to the virtual table domain.
+  end: number // last row index (exclusive)
 }
 
 interface RowsAndColumnsContextType {
   columnsParameters?: ColumnParameters[]
-  rowsRangeWithPadding?: RowsRangeWithPadding
-  setRowsRange?: (rowsRange: RowsRange | undefined) => void
+  visibleRowsRange?: RowsRange // range of rows visible in the viewport
+  fetchedRowsRange?: RowsRange // range of rows fetched from the data source (including overscan)
+  renderedRowsRange?: RowsRange // range of rows rendered in the DOM as table rows (including padding and overscan)
+  setVisibleRowsRange?: (rowsRange: RowsRange | undefined) => void
 }
 
 export const defaultRowsAndColumnsContext: RowsAndColumnsContextType = {}

src/providers/RowsAndColumnsProvider.tsx

@@ -1,7 +1,7 @@
 import type { ReactNode } from 'react'
 import { useContext, useMemo, useState } from 'react'
 
-import { defaultPadding } from '../components/HighTable/constants.js'
+import { defaultOverscan, defaultPadding } from '../components/HighTable/constants.js'
 import { ColumnParametersContext } from '../contexts/ColumnParametersContext.js'
 import { ColumnVisibilityStatesContext } from '../contexts/ColumnVisibilityStatesContext.js'
 import { DataContext } from '../contexts/DataContext.js'
@@ -11,15 +11,16 @@ import type { RowsRange } from '../contexts/RowsAndColumnsContext.js'
 import { RowsAndColumnsContext } from '../contexts/RowsAndColumnsContext.js'
 
 export interface RowsAndColumnsProviderProps {
-  padding?: number
+  overscan?: number // number of rows to fetch outside of the viewport
+  padding?: number // number of empty placeholder rows to render beyond the fetched data range
 }
 
 type Props = {
   children: ReactNode
 } & RowsAndColumnsProviderProps
 
-export function RowsAndColumnsProvider({ padding = defaultPadding, children }: Props) {
-  const [rowsRange, setRowsRange] = useState<RowsRange | undefined>(undefined)
+export function RowsAndColumnsProvider({ padding = defaultPadding, overscan = defaultOverscan, children }: Props) {
+  const [visibleRowsRange, setVisibleRowsRange] = useState<RowsRange | undefined>(undefined)
 
   const { onError } = useContext(ErrorContext)
   const { data, numRows } = useContext(DataContext)
@@ -33,12 +34,29 @@ export function RowsAndColumnsProvider({ padding = defaultPadding, children }: P
     })
   }, [allColumnsParameters, isHiddenColumn])
 
+  const fetchedRowsRange = useMemo(() => {
+    if (!visibleRowsRange) return undefined
+
+    return {
+      start: Math.max(0, visibleRowsRange.start - overscan),
+      end: Math.min(numRows, visibleRowsRange.end + overscan),
+    }
+  }, [visibleRowsRange, numRows, overscan])
+  const renderedRowsRange = useMemo(() => {
+    if (!fetchedRowsRange) return undefined
+
+    return {
+      start: Math.max(0, fetchedRowsRange.start - padding),
+      end: Math.min(numRows, fetchedRowsRange.end + padding),
+    }
+  }, [fetchedRowsRange, numRows, padding])
+
   const fetchOptions = useMemo(() => ({
     orderBy,
     columnsParameters,
-    rowsRange,
+    fetchedRowsRange,
     abortController: new AbortController(),
-  }), [orderBy, columnsParameters, rowsRange])
+  }), [orderBy, columnsParameters, fetchedRowsRange])
   const [lastFetchOptions, setLastFetchOptions] = useState(fetchOptions)
 
   // fetch the rows if needed
@@ -47,15 +65,15 @@ export function RowsAndColumnsProvider({ padding = defaultPadding, children }: P
   // No need for useEffect
   if (
     lastFetchOptions.orderBy !== orderBy
-    || lastFetchOptions.rowsRange !== rowsRange
+    || lastFetchOptions.fetchedRowsRange !== fetchedRowsRange
     || lastFetchOptions.columnsParameters !== columnsParameters
   ) {
     lastFetchOptions.abortController.abort()
     setLastFetchOptions(fetchOptions)
-    if (data.fetch !== undefined && rowsRange !== undefined) {
+    if (data.fetch !== undefined && fetchedRowsRange !== undefined) {
       data.fetch({
-        rowStart: rowsRange.start,
-        rowEnd: rowsRange.end,
+        rowStart: fetchedRowsRange.start,
+        rowEnd: fetchedRowsRange.end,
         columns: columnsParameters.map(({ name }) => name),
         orderBy,
         signal: fetchOptions.abortController.signal,
@@ -69,23 +87,13 @@ export function RowsAndColumnsProvider({ padding = defaultPadding, children }: P
     }
   }
 
-  const rowsRangeWithPadding = useMemo(() => {
-    if (!rowsRange) return undefined
-
-    const startPadding = Math.max(rowsRange.start - padding, 0)
-    const endPadding = Math.min(rowsRange.end + padding, numRows)
-    return {
-      ...rowsRange,
-      startPadding,
-      endPadding,
-    }
-  }, [rowsRange, numRows, padding])
-
   const value = useMemo(() => ({
     columnsParameters,
-    rowsRangeWithPadding,
-    setRowsRange,
-  }), [columnsParameters, rowsRangeWithPadding, setRowsRange])
+    visibleRowsRange,
+    renderedRowsRange,
+    fetchedRowsRange,
+    setVisibleRowsRange,
+  }), [columnsParameters, fetchedRowsRange, setVisibleRowsRange, renderedRowsRange, visibleRowsRange])
 
   return (
     <RowsAndColumnsContext.Provider value={value}>