add 'load what you can' button to failed tables; decode keys in json

dt · dt · commit e3985f4df310 · 2025-11-19T16:16:05.000Z
diff --git a/src/components/ErrorViewer.tsx b/src/components/ErrorViewer.tsx
@@ -5,6 +5,7 @@ interface ErrorViewerProps {
   tabId: string;
   error: string;
   sourceFile: string;
+  sourceFileSize?: number;
   tableName: string;
   fullTableName?: string;
   isPreLoadError?: boolean;
@@ -16,6 +17,7 @@ function ErrorViewer({
   tabId,
   error,
   sourceFile,
+  sourceFileSize,
   tableName,
   fullTableName,
   isPreLoadError,
@@ -84,6 +86,52 @@ function ErrorViewer({
     }
   };
 
+  const handleLoadPartialData = async () => {
+    if (!state.workerManager) {
+      return;
+    }
+
+    const tableNameForLoading = fullTableName || tableName;
+    // Get the source file path (non-.err file)
+    const sourceFilePath = sourceFile.replace(/\.err\.txt$/, '');
+
+    setIsLoading(true);
+    setLoadError(null);
+    try {
+      // Load the table with ignore_errors flag
+      await state.workerManager.loadSingleTable({
+        name: tableNameForLoading,
+        path: sourceFilePath,
+        size: sourceFileSize || 0,
+        originalName: tableName,
+        isError: false,
+        ignoreErrors: true, // NEW: Tell the loader to use relaxed mode
+      });
+
+      // Close this error tab and open the SQL tab for the loaded table
+      dispatch({ type: "CLOSE_TAB", id: tabId });
+      setTimeout(() => {
+        dispatch({
+          type: "OPEN_TAB",
+          tab: {
+            kind: "sql",
+            id: `sql-${tableNameForLoading}`,
+            title: tableNameForLoading,
+            query: `SELECT * FROM "${tableNameForLoading}" LIMIT 100`,
+            sourceTable: tableNameForLoading,
+            isCustomQuery: false,
+          },
+        });
+      }, 100);
+    } catch (err) {
+      console.error("Failed to load partial data:", err);
+      const errorMessage = err instanceof Error ? err.message : String(err);
+      setLoadError(errorMessage);
+    } finally {
+      setIsLoading(false);
+    }
+  };
+
   // Parse the error message to extract useful information
   const parseError = (errorMsg: string) => {
     // Look for line number in error
@@ -103,6 +151,7 @@ function ErrorViewer({
   if (isPreLoadError) {
     const isSingleFile = !errorFiles || errorFiles.length === 1;
     const hasAvailableFiles = availableFiles && availableFiles.length > 0;
+    const hasPartialData = isSingleFile && sourceFileSize && sourceFileSize > 0;
 
     // Helper to format file sizes
     const formatSize = (bytes?: number) => {
@@ -149,6 +198,33 @@ function ErrorViewer({
             </ul>
           </div>
 
+          {hasPartialData && !hasAvailableFiles && (
+            <div className="error-section">
+              <h3>Partial Data Available</h3>
+              <p style={{ marginBottom: "12px" }}>
+                The source file contains {formatSize(sourceFileSize)} of data. You can attempt to load what can be parsed with relaxed error handling.
+              </p>
+              <p style={{ marginBottom: "12px", color: "var(--text-muted)" }}>
+                <strong>Note:</strong> This will skip malformed rows and may result in incomplete data.
+              </p>
+              <button
+                className="btn btn-primary"
+                onClick={handleLoadPartialData}
+                disabled={isLoading}
+                style={{ marginTop: "16px" }}
+              >
+                {isLoading ? "Loading..." : `Load Partial Data (${formatSize(sourceFileSize)})`}
+              </button>
+
+              {loadError && (
+                <div style={{ marginTop: "16px", padding: "12px", backgroundColor: "var(--error-bg, #3c1f1f)", border: "1px solid var(--error-border, #f44336)", borderRadius: "4px" }}>
+                  <h4 style={{ margin: "0 0 8px 0", color: "var(--error-color, #f44336)" }}>Failed to Load Partial Data</h4>
+                  <pre style={{ margin: 0, whiteSpace: "pre-wrap", wordBreak: "break-word", fontSize: "0.9em" }}>{loadError}</pre>
+                </div>
+              )}
+            </div>
+          )}
+
           {hasAvailableFiles && (
             <div className="error-section">
               <h3>Available Files ({availableFiles.length} nodes)</h3>
diff --git a/src/components/MainPanel.tsx b/src/components/MainPanel.tsx
@@ -113,6 +113,7 @@ function MainPanel({ style }: { style?: React.CSSProperties }) {
                     tabId={tab.id}
                     error={tab.error}
                     sourceFile={tab.sourceFile}
+                    sourceFileSize={tab.sourceFileSize}
                     tableName={tab.tableName}
                     fullTableName={tab.fullTableName}
                     isPreLoadError={tab.isPreLoadError}
diff --git a/src/components/sidebar/TablesView.tsx b/src/components/sidebar/TablesView.tsx
@@ -415,6 +415,15 @@ function TablesView() {
         [{ path: table.sourceFile, nodeId: table.nodeId ?? 0, size: table.size ?? 0, isError: true }];
       const availableFiles = table.nodeFiles?.filter(f => !f.isError) || [];
 
+      // For single-file errors, try to find the source (non-.err) file and its size
+      let sourceFileSize: number | undefined;
+      if (!table.nodeFiles || table.nodeFiles.length <= 1) {
+        // Single file error - try to find the corresponding non-.err file
+        const sourceFilePath = table.sourceFile.replace(/\.err\.txt$/, '');
+        const sourceFileEntry = state.filesIndex[sourceFilePath];
+        sourceFileSize = sourceFileEntry?.size;
+      }
+
       dispatch({
         type: "OPEN_TAB",
         tab: {
@@ -423,6 +432,7 @@ function TablesView() {
           title: `Error: ${baseName}`,
           error: "", // Will be populated by ErrorViewer
           sourceFile: table.sourceFile,
+          sourceFileSize,
           tableName: baseName,
           fullTableName: table.name, // Full name with _by_node suffix if applicable
           isPreLoadError: true,
diff --git a/src/crdb/batchProcessor.ts b/src/crdb/batchProcessor.ts
@@ -21,6 +21,7 @@ interface BatchProcessOptions {
   sourcePath?: string;
   nodeId?: number;
   forceInsert?: boolean; // Force INSERT mode (table already exists)
+  ignoreErrors?: boolean; // Use relaxed CSV parsing to load partial data
 }
 
 const BATCH_SIZE_ROWS = 10000; // Max rows per batch
@@ -129,6 +130,7 @@ function tryReplaceWithPrettyKey(value: string): string {
     return value;
   }
 
+  // Handle hex keys (start with \x)
   if (value === "\\x" || value.startsWith("\\x")) {
     try {
       const decoded = prettyKey(value);
@@ -138,16 +140,74 @@ function tryReplaceWithPrettyKey(value: string): string {
     }
   }
 
+  // Handle base64-encoded keys (like oA==, oQ==, etc.)
+  // Check if it looks like base64: only contains base64 chars and correct padding
+  if (/^[A-Za-z0-9+/]*(=|==)?$/.test(value) && value.length % 4 === 0 && value.length > 0) {
+    try {
+      // Convert base64 to hex
+      const binaryString = atob(value);
+      const bytes = new Uint8Array(binaryString.length);
+      for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+      }
+      const hexStr = Array.from(bytes)
+        .map((b) => b.toString(16).padStart(2, "0"))
+        .join("");
+
+      const decoded = prettyKey(hexStr);
+      // Only use the decoded version if it's different from the hex
+      if (decoded.pretty !== hexStr) {
+        return decoded.pretty;
+      }
+    } catch {
+      // If decoding fails, return original value
+    }
+  }
+
   return value;
 }
 
+// Recursively decode base64 keys in JSON objects
+function decodeKeysInJson(obj: unknown): unknown {
+  if (typeof obj === "string") {
+    // If it's a string that looks like base64, try to decode it as a key
+    return tryReplaceWithPrettyKey(obj);
+  } else if (Array.isArray(obj)) {
+    return obj.map(decodeKeysInJson);
+  } else if (obj !== null && typeof obj === "object") {
+    const result: Record<string, unknown> = {};
+    for (const [key, value] of Object.entries(obj)) {
+      // Decode values in key-related fields
+      if (key.toLowerCase().includes("key") || key === "start" || key === "end") {
+        result[key] = typeof value === "string" ? tryReplaceWithPrettyKey(value) : decodeKeysInJson(value);
+      } else {
+        result[key] = decodeKeysInJson(value);
+      }
+    }
+    return result;
+  }
+  return obj;
+}
+
+function decodeJsonKeys(jsonString: string): string {
+  try {
+    const parsed = JSON.parse(jsonString);
+    const decoded = decodeKeysInJson(parsed);
+    return JSON.stringify(decoded);
+  } catch {
+    // If parsing fails, return original
+    return jsonString;
+  }
+}
+
 async function processRow(
   row: string[],
   headers: string[],
   rowIndex: number,
   options: {
     tableName: string;
     keyColumns: Set<number>;
+    jsonKeyColumns: Set<number>;
     protoColumns: Map<number, string | null>;
     infoKeyColumnIndex: number;
     decodeKeys: boolean;
@@ -157,6 +217,18 @@ async function processRow(
 ): Promise<string[]> {
   const processedRow = await Promise.all(
     row.map(async (value, colIndex) => {
+      // Transform JSON columns that contain base64 keys
+      if (options.jsonKeyColumns.has(colIndex) && options.decodeKeys) {
+        if (value === "\\N" || value === "NULL" || !value) {
+          return value;
+        }
+        // Check if it looks like JSON
+        if (value.startsWith("{") || value.startsWith("[")) {
+          return decodeJsonKeys(value);
+        }
+        return value;
+      }
+
       // Transform key columns
       if (options.keyColumns.has(colIndex) && options.decodeKeys) {
         if (value === "\\N" || value === "NULL") {
@@ -242,7 +314,7 @@ export async function preprocessAndLoadInBatches(
   content: string,
   options: BatchProcessOptions
 ): Promise<number> {
-  const { conn, db, tableName, delimiter, sourcePath, nodeId } = options;
+  const { conn, db, tableName, delimiter, sourcePath, nodeId, ignoreErrors } = options;
 
   // Process content in a streaming fashion
   let position = 0;
@@ -266,6 +338,7 @@ export async function preprocessAndLoadInBatches(
   // Identify columns that need special processing
   const keyColumns = new Set<number>();
   const protoColumns = new Map<number, string | null>();
+  const jsonKeyColumns = new Set<number>(); // JSON columns that may contain base64 keys
   let infoKeyColumnIndex = -1;
 
   headers.forEach((header, index) => {
@@ -276,7 +349,14 @@ export async function preprocessAndLoadInBatches(
       keyColumns.add(index);
     }
 
-    // Check for proto columns
+    // Check for JSON columns that may contain encoded keys (e.g., rangelog.info)
+    // These are already JSON from CRDB, not protos that need decoding
+    if (options.decodeKeys && columnName === "info" &&
+        (tableName.includes("rangelog") || tableName.includes("range_log"))) {
+      jsonKeyColumns.add(index);
+    }
+
+    // Check for proto columns (hex-encoded that need proto->JSON conversion)
     if (options.decodeProtos && options.protoDecoder) {
       if (columnName === "config" || columnName === "descriptor" ||
           columnName === "payload" || columnName === "progress" || columnName === "value") {
@@ -347,6 +427,7 @@ export async function preprocessAndLoadInBatches(
     const processedRow = await processRow(row, headers, lineNumber, {
       tableName,
       keyColumns,
+      jsonKeyColumns,
       protoColumns,
       infoKeyColumnIndex,
       decodeKeys: options.decodeKeys,
@@ -383,6 +464,7 @@ export async function preprocessAndLoadInBatches(
           nodeId, // Always pass nodeId for both CREATE and INSERT
           typeHints,
           headers: !tableCreated ? headers : undefined,
+          ignoreErrors,
         });
 
         await conn.query(sql);
@@ -422,6 +504,7 @@ export async function preprocessAndLoadInBatches(
         nodeId, // Always pass nodeId for both CREATE and INSERT
         typeHints,
         headers: !tableCreated ? headers : undefined,
+        ignoreErrors,
       });
 
       await conn.query(sql);
diff --git a/src/crdb/csvUtils.ts b/src/crdb/csvUtils.ts
@@ -9,10 +9,11 @@ export interface CsvReadOptions {
   nodeId?: number;
   typeHints?: Map<string, string>;
   headers?: string[];
+  ignoreErrors?: boolean; // Use relaxed CSV parsing to skip malformed rows
 }
 
 export function generateCsvReadSql(options: CsvReadOptions): string {
-  const { fileName, tableName, operation, nodeId, typeHints, headers } = options;
+  const { fileName, tableName, operation, nodeId, typeHints, headers, ignoreErrors } = options;
   const quotedTableName = `"${tableName}"`;
 
   // Build column specification if we have type hints and headers
@@ -35,7 +36,7 @@ export function generateCsvReadSql(options: CsvReadOptions): string {
     escape = '"',
     header = true,
     nullstr = ['NULL', '\\N'],
-    max_line_size = 33554432${columnsClause}
+    max_line_size = 33554432${ignoreErrors ? ',\n    ignore_errors = true' : ''}${columnsClause}
   `;
 
   if (operation === 'create') {
diff --git a/src/state/types.ts b/src/state/types.ts
@@ -37,6 +37,7 @@ export type ViewerTab =
       title: string;
       error: string;
       sourceFile: ZipEntryId;
+      sourceFileSize?: number; // Size of the non-.err source file (for partial loading)
       tableName: string; // Display name (e.g., "crdb_internal.node_build_info")
       fullTableName?: string; // Full table name for loading (e.g., "crdb_internal.node_build_info_by_node")
       isPreLoadError?: boolean; // true for .err.txt files, undefined for DuckDB load errors
@@ -169,6 +170,7 @@ export interface TableData {
   nodeId?: number;
   originalName?: string;
   isError?: boolean;
+  ignoreErrors?: boolean; // Use relaxed CSV parsing to load partial data
   loaded?: boolean;
   loading?: boolean;
   sourceFile?: string;
diff --git a/src/workers/db.worker.ts b/src/workers/db.worker.ts
