feat: add excludedColumns functionality

Signed-off-by: Tomáš Dvořák <toomas2d@gmail.com>

Author: Tomas2D

README.md

@@ -68,6 +68,7 @@ interface ParserSettings {
   colFilter?: (elText: string[], index: number) => string; // (default: (txt: string) => txt.join(' '))
   colParser?: (value: string, formattedIndex: number, getColumnIndex: GetColumnIndexType) => string; // (default: (txt: string) => txt.trim())
   optionalColNames?: string[]; // (default: [])
+  excludedColumns?: (rows: string[][], getColumnIndex: GetColumnIndexType) => string[]; // (default: undefined)
 };
 ```
 
@@ -82,8 +83,9 @@ interface ParserSettings {
 7. Run `rowTransform` function for each row.
 8. Group results into buckets (`groupBy.cols`) property and pick the aggregated rows.
 9. Add processed row to a temp array result. 
-10. Add `header` column if `withHeader` property is `true`.
-11. Merge partial results and return them.
+10. Run `excludedColumns` and exclude retrieved columns from the rows and the `header`. 
+11. Add `header` row (if `withHeader` property is `true`).
+12. Merge partial results and return them.
 
 ## Examples
 

src/helpers.ts

@@ -77,7 +77,10 @@ export async function getColumnsInfo(
   const excludedKeyIndexes: number[] = [];
   const colKeyToIndexWithExcluded = new Map<string, number>();
   const colIndexToKeyWithExcluded = new Map<number, string>();
-  extraColsMapper(allowedColNamesKeys, 'colName').forEach((key, index) => {
+
+  const allColumnKeys = extraColsMapper(allowedColNamesKeys, 'colName');
+
+  allColumnKeys.forEach((key, index) => {
     colKeyToIndexWithExcluded.set(key, index);
     colIndexToKeyWithExcluded.set(index, key);
 
@@ -99,13 +102,19 @@ export async function getColumnsInfo(
     return index;
   };
 
-  const getColumnName = (colIndex: number) => {
-    const value = colIndexToKeyWithExcluded.get(colIndex);
-    if (value === undefined) {
-      throw new InvalidColumnError(`Column with index '${colIndex}' does not exist!`);
-    }
+  const updateExcludedColumns = (colNames: string[]) => {
+    colNames.forEach((colName) => {
+      const index = getColumnIndex(colName);
+      if (!excludedKeyIndexes.includes(index)) {
+        excludedKeyIndexes.push(index);
+      }
+    });
+  };
 
-    return value;
+  const getOutputColumnKeys = () => {
+    return allColumnKeys
+      .filter((_, index) => !excludedKeyIndexes.includes(index))
+      .map((colName) => settings.allowedColNames[colName] || colName);
   };
 
   return {
@@ -115,7 +124,8 @@ export async function getColumnsInfo(
     },
     missingColNames: Object.values(missingColNames),
     getColumnIndex,
-    getColumnName,
+    updateExcludedColumns,
+    getOutputColumnKeys,
   };
 }
 

src/parseTable.ts

@@ -104,11 +104,8 @@ export function parseTableFactory(settings: FullParserSettings) {
       return [];
     }
 
-    const { indexes, getColumnIndex, getColumnName, missingColNames } = await getColumnsInfo(
-      settings,
-      header.el,
-      extraColsMapper,
-    );
+    const { indexes, getColumnIndex, missingColNames, updateExcludedColumns, getOutputColumnKeys } =
+      await getColumnsInfo(settings, header.el, extraColsMapper);
 
     let parsedRows = await new PipelineExecutor<string[][], string[][]>(
       await getRowsData(table, header.bodyRowsOffset, indexes.allowed),
@@ -124,16 +121,26 @@ export function parseTableFactory(settings: FullParserSettings) {
       parsedRows = groupBy(parsedRows, settings.groupBy, getColumnIndex);
     }
 
+    if (settings.excludedColumns) {
+      const excludedColNames = settings.excludedColumns(parsedRows, getColumnIndex) ?? [];
+      updateExcludedColumns(excludedColNames);
+    }
+
     if (addHeader) {
       const headerRow = getOutputHeaderRow(missingColNames);
       parsedRows.unshift(headerRow);
     }
 
     const rowOutputMapper = settings.rowValuesAsObject
-      ? mappers.asObject(getColumnName)
+      ? mappers.asObject(
+          (() => {
+            const outputColumnKeys = getOutputColumnKeys();
+            return (index) => outputColumnKeys[index];
+          })(),
+        )
       : settings.rowValuesAsArray
-        ? mappers.asArray()
-        : mappers.asCsv(settings.csvSeparator);
+      ? mappers.asArray()
+      : mappers.asCsv(settings.csvSeparator);
 
     return new PipelineExecutor<string[][], ReturnType<typeof rowOutputMapper>[]>(parsedRows)
       .addMap((row) => row.filter((_, index) => !indexes.excluded.includes(index)))

src/settings.ts

@@ -28,6 +28,7 @@ export const defaultSettings: ParserSettingsOptional = {
   headerRowsCellSelector: 'td,th',
   bodyRowsSelector: 'tbody tr',
   bodyRowsCellSelector: 'td',
+  excludedColumns: undefined,
 };
 
 export function preprocessSettings(options: ParserSettings): FullParserSettings {

src/types.ts

@@ -56,6 +56,7 @@ export type ParserSettingsOptional = {
   headerRowsCellSelector: string;
   bodyRowsSelector: string;
   bodyRowsCellSelector: string;
+  excludedColumns: (rows: string[][], getColumnIndex: GetColumnIndexType) => string[];
 };
 
 export interface ParserSettings extends Partial<ParserSettingsOptional> {

test/assets/3.html

@@ -0,0 +1,31 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Example #3</title>
+</head>
+<body>
+
+<table id="table-overview">
+  <thead>
+  <tr>
+    <th>A</th>
+    <th><input type="checkbox" checked></th>
+    <th>C</th>
+  </tr>
+  </thead>
+  <tbody>
+  <tr>
+    <td>A1</td>
+    <td>B1</td>
+    <td><img src='#' alt='image'>C1</td>
+  </tr>
+  <tr>
+    <td><a href='#'>A1</a></td>
+    <td><input type="checkbox" checked></td>
+    <td>C1</td>
+  </tr>
+  </tbody>
+</table>
+
+</body>
+</html>

test/index.test.ts

@@ -4,8 +4,6 @@ import { createServer, getBaseUrl } from './createServer';
 import { Browser, launch, Page } from 'puppeteer';
 import { tableParser, RowValidationPolicy } from '../src';
 
-jest.setTimeout(60 * 1000 * 1000);
-
 describe('Basic parsing', () => {
   let server: Server;
   let browser: Browser;
@@ -14,7 +12,7 @@ describe('Basic parsing', () => {
   beforeAll(async () => {
     server = await createServer();
     browser = await launch({
-      headless: 'new',
+      headless: true,
     });
     page = await browser.newPage();
   });
@@ -137,8 +135,8 @@ describe('Basic parsing', () => {
     `);
   });
 
-  it('Throw error with invalid options', () => {
-    expect(
+  it('Throw error with invalid options', async () => {
+    await expect(
       tableParser(page, {
         selector: 'table',
         asArray: false,
@@ -150,7 +148,7 @@ describe('Basic parsing', () => {
       }),
     ).rejects.toThrowError();
 
-    expect(
+    await expect(
       tableParser(page, {
         selector: 'table',
         // @ts-expect-error intended
@@ -163,7 +161,7 @@ describe('Basic parsing', () => {
       }),
     ).rejects.toThrowError();
 
-    expect(
+    await expect(
       tableParser(page, {
         selector: 'table',
         rowValuesAsObject: false,
@@ -176,8 +174,8 @@ describe('Basic parsing', () => {
     ).rejects.toThrowError();
   });
 
-  it('Throw error when specified optional column which does not exists', () => {
-    expect(
+  it('Throw error when specified optional column which does not exists', async () => {
+    await expect(
       tableParser(page, {
         selector: 'table',
         allowedColNames: {
@@ -553,4 +551,58 @@ describe('Basic parsing', () => {
     expect(data.length).toBe(1);
     expect(data[0]).toBe('name,age');
   });
+
+  it('Handles non-text elements', async () => {
+    await page.goto(`${getBaseUrl()}/3.html`);
+
+    const data = await tableParser(page, {
+      selector: '#table-overview',
+      asArray: false,
+      allowedColNames: {
+        'A': 'A',
+        '': 'B',
+        'C': 'C',
+      },
+      colParser: (value, x) => {
+        console.info({ value, x });
+        return value.trim();
+      },
+    });
+
+    expect(data).toMatchInlineSnapshot(`
+      "A;B;C
+      A1;B1;C1
+      A1;;C1"
+    `);
+  });
+
+  it('Exclude columns', async () => {
+    await page.goto(`${getBaseUrl()}/1.html`);
+
+    const data = await tableParser(page, {
+      selector: 'table',
+      allowedColNames: {
+        'Car Name': 'car',
+        'Horse Powers': 'hp',
+        'Manufacture Year': 'year',
+      },
+      optionalColNames: ['year'],
+      rowValidationPolicy: RowValidationPolicy.EXACT_MATCH,
+      colParser: (value) => value.trim(),
+      excludedColumns: (rows, getColumnIndex) => {
+        expect(rows.length).toBe(4);
+        expect(getColumnIndex('car')).toBe(0);
+        expect(getColumnIndex('hp')).toBe(1);
+        expect(getColumnIndex('year')).toBe(2);
+        return ['year'];
+      },
+    });
+
+    expect(data).toMatchInlineSnapshot(`
+      "car;hp
+      Audi S5;332
+      Alfa Romeo Giulia;500
+      BMW X3;215
+      Skoda Octavia;120"`);
+  });
 });