feat: Enhance nested foreach loops by supporting custom index placeholders and preserving parent context in results.

lablnet · lablnet · commit 39004b67d7dc · 2025-12-26T10:04:35.000+05:00
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "stepwright",
-    "version": "1.0.8",
+    "version": "1.0.9",
     "description": "A powerful web scraping library built with Playwright",
     "main": "dist/index.js",
     "types": "dist/index.d.ts",
diff --git a/src/step-executor.ts b/src/step-executor.ts
@@ -141,7 +141,17 @@ export async function executeStep(
     case 'click': {
       try {
         // Check if element exists first
-        const locator = locatorFor(page, step.object_type as SelectorType | undefined, step.object ?? '');
+        const locator = scopeLocator
+          ? locatorFor(
+              scopeLocator as any,
+              step.object_type as SelectorType | undefined,
+              step.object ?? ''
+            )
+          : locatorFor(
+              page,
+              step.object_type as SelectorType | undefined,
+              step.object ?? ''
+            );
         const count = await locator.count();
         
         if (count === 0) {
@@ -267,7 +277,17 @@ export async function executeStep(
       if (!step.subSteps || step.subSteps.length === 0) {
         throw new Error('foreach step requires subSteps');
       }
-      const locatorAll = locatorFor(page, step.object_type as SelectorType | undefined, step.object);
+      const locatorAll = scopeLocator
+        ? locatorFor(
+            scopeLocator as any,
+            step.object_type as SelectorType | undefined,
+            step.object
+          )
+        : locatorFor(
+            page,
+            step.object_type as SelectorType | undefined,
+            step.object
+          );
       try {
         await locatorAll.first().waitFor({ state: 'attached', timeout: step.wait ?? 5000 });
       } catch {}
@@ -283,11 +303,20 @@ export async function executeStep(
         }
 
         // Create a separate collector for each iteration
+        // Initialize with parent context (non-item_* keys) to preserve metadata
         const itemCollector: Record<string, any> = {};
+        Object.keys(collector).forEach(k => {
+          if (!k.startsWith('item_')) {
+            itemCollector[k] = collector[k];
+          }
+        });
+
+        // Use the specified index key or default to 'i'
+        const indexKey = step.index_key || 'i';
 
         // For each subStep clone and replace placeholders
         for (const s of step.subSteps) {
-          const cloned = cloneStepWithIndex(s, idx);
+          const cloned = cloneStepWithIndex(s, idx, indexKey);
           try {
             await executeStep(page, cloned, itemCollector, onResult, current);
           } catch (err: any) {
@@ -307,8 +336,6 @@ export async function executeStep(
           );
 
           // Emit the result immediately for streaming
-          // We need to access the onResult callback from the parent context
-          // This is a bit of a hack, but it works for immediate streaming
           if ((global as any).onResultCallback) {
             try {
               const flattenedResult = flattenNestedForeachResults(itemCollector);
@@ -329,7 +356,17 @@ export async function executeStep(
       
       try {
         // locate link and check if it exists
-        const linkLoc = locatorFor(page, step.object_type as SelectorType | undefined, step.object);
+        const linkLoc = scopeLocator
+          ? locatorFor(
+              scopeLocator as any,
+              step.object_type as SelectorType | undefined,
+              step.object
+            )
+          : locatorFor(
+              page,
+              step.object_type as SelectorType | undefined,
+              step.object
+            );
         const count = await linkLoc.count();
         
         if (count === 0) {
@@ -365,8 +402,13 @@ export async function executeStep(
           await newPage.waitForLoadState('networkidle');
         }
 
-        // Pass the parent collector data to subSteps so they can access meeting_title, meeting_date, etc.
-        const innerCollected: Record<string, any> = { ...collector };
+        // Pass only the parent context (non-item keys) to subSteps
+        const innerCollected: Record<string, any> = {};
+        Object.keys(collector).forEach(k => {
+          if (!k.startsWith('item_')) {
+            innerCollected[k] = collector[k];
+          }
+        });
         for (const s of step.subSteps) {
           const cloned: BaseStep = { ...s };
           try {
@@ -673,7 +715,17 @@ export async function executeStep(
         // If object is provided, click it first (for backward compatibility)
         if (step.object) {
           try {
-            const locator = locatorFor(page, step.object_type as SelectorType | undefined, step.object);
+            const locator = scopeLocator
+              ? locatorFor(
+                  scopeLocator as any,
+                  step.object_type as SelectorType | undefined,
+                  step.object
+                )
+              : locatorFor(
+                  page,
+                  step.object_type as SelectorType | undefined,
+                  step.object
+                );
             const count = await locator.count();
             
             if (count > 0) {
@@ -743,11 +795,17 @@ export async function executeStep(
 
       try {
         // 1) Find the link and get its href
-        const link = locatorFor(
-          page,
-          step.object_type as SelectorType | undefined,
-          step.object
-        );
+        const link = scopeLocator
+          ? locatorFor(
+              scopeLocator as any,
+              step.object_type as SelectorType | undefined,
+              step.object
+            )
+          : locatorFor(
+              page,
+              step.object_type as SelectorType | undefined,
+              step.object
+            );
         const present = (await link.count()) > 0;
         if (!present) {
           console.log(`   ⚠️  PDF link not found: ${step.object}`);
diff --git a/src/types.ts b/src/types.ts
@@ -19,6 +19,7 @@ export interface BaseStep {
   terminateonerror?: boolean;
   subSteps?: BaseStep[]; // for foreach/open
   autoScroll?: boolean; // for foreach action - controls automatic scrolling (default: true)
+  index_key?: string; // custom index placeholder for loops (e.g., 'j', 'k')
 }
 
 export interface PaginationConfig {
diff --git a/src/utils.ts b/src/utils.ts
@@ -18,9 +18,11 @@ import { SelectorType } from './types';
  * @since v1.0.0
  * @company Framework Island
  */
-export function replaceIndexPlaceholders(text: string | undefined, i: number): string | undefined {
+export function replaceIndexPlaceholders(text: string | undefined, i: number, char: string = 'i'): string | undefined {
   if (!text) return text;
-  return text.replace(/\{\{\s*i\s*\}\}/g, i.toString()).replace(/\{\{\s*i_plus1\s*\}\}/g, (i+1).toString());
+  const regex = new RegExp(`\\{\\{\\s*${char}\\s*\\}\\}`, 'g');
+  const plus1Regex = new RegExp(`\\{\\{\\s*${char}_plus1\\s*\\}\\}`, 'g');
+  return text.replace(regex, i.toString()).replace(plus1Regex, (i + 1).toString());
 }
 
 /**
@@ -97,13 +99,13 @@ export function locatorFor(page: Page, type: SelectorType | undefined, selector:
  * @since v1.0.0
  * @company Framework Island
  */
-export function cloneStepWithIndex(step: import('./types').BaseStep, idx: number): import('./types').BaseStep {
+export function cloneStepWithIndex(step: import('./types').BaseStep, idx: number, char: string = 'i'): import('./types').BaseStep {
   const cloned: import('./types').BaseStep = { ...step };
-  cloned.object = replaceIndexPlaceholders(cloned.object, idx);
-  cloned.value = replaceIndexPlaceholders(cloned.value, idx);
-  cloned.key = replaceIndexPlaceholders(cloned.key, idx);
+  cloned.object = replaceIndexPlaceholders(cloned.object, idx, char);
+  cloned.value = replaceIndexPlaceholders(cloned.value, idx, char);
+  cloned.key = replaceIndexPlaceholders(cloned.key, idx, char);
   if (cloned.subSteps && cloned.subSteps.length > 0) {
-    cloned.subSteps = cloned.subSteps.map((sub) => cloneStepWithIndex(sub, idx));
+    cloned.subSteps = cloned.subSteps.map((sub) => cloneStepWithIndex(sub, idx, char));
   }
   return cloned;
 }
@@ -121,23 +123,34 @@ export function cloneStepWithIndex(step: import('./types').BaseStep, idx: number
  * @company Framework Island
  */
 export function flattenNestedForeachResults(item: Record<string, any>): any {
+  // Extract context keys (keys NOT starting with item_)
+  const context: Record<string, any> = {};
+  Object.keys(item).forEach(k => {
+    if (!k.startsWith('item_')) {
+      context[k] = item[k];
+    }
+  });
+
   // Check if item contains nested item_* keys (from nested foreach)
   const nestedItemKeys = Object.keys(item).filter(k => k.startsWith('item_'));
   if (nestedItemKeys.length > 0) {
     // Flatten nested items into an array
     const flattenedItems: any[] = [];
-    for (const k of nestedItemKeys.sort((a, b) => {
+    const sortedKeys = nestedItemKeys.sort((a, b) => {
       const aIdx = parseInt(a.split('_')[1]);
       const bIdx = parseInt(b.split('_')[1]);
       return aIdx - bIdx;
-    })) {
+    });
+
+    for (const k of sortedKeys) {
       if (item[k] && Object.keys(item[k]).length > 0) {
-        flattenedItems.push(item[k]);
+        // Merge context into the nested item
+        flattenedItems.push({ ...context, ...item[k] });
       }
     }
     return flattenedItems;
   } else {
-    // No nested items, return item as is
+    // No nested items, return item with context (already in item)
     return item;
   }
 }
diff --git a/test/nested-loop.html b/test/nested-loop.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Nested Loop Test</title>
+</head>
+<body>
+  <div class="parent">
+    <h1>Meeting 1</h1>
+    <div class="child">Attachment 1.1</div>
+    <div class="child">Attachment 1.2</div>
+  </div>
+  <div class="parent">
+    <h1>Meeting 2</h1>
+    <div class="child">Attachment 2.1</div>
+  </div>
+</body>
+</html>
diff --git a/test/nested-loop.test.ts b/test/nested-loop.test.ts
@@ -0,0 +1,139 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { getBrowser, navigate } from '../src/scraper';
+import { executeStep } from '../src/step-executor';
+import { Browser, Page } from 'playwright';
+import path from 'path';
+import { BaseStep } from '../src/types';
+
+describe('Nested Foreach Loops', () => {
+  let browser: Browser;
+  let page: Page;
+  const testPagePath = path.join(__dirname, 'nested-loop.html');
+  const testPageUrl = `file://${testPagePath}`;
+
+  beforeAll(async () => {
+    browser = await getBrowser({ headless: true });
+    page = await browser.newPage();
+  });
+
+  afterAll(async () => {
+    await browser.close();
+  });
+
+  it('should support nested loops and preserve parent context', async () => {
+    await navigate(page, testPageUrl);
+
+    const collector: Record<string, any> = {};
+    const nestedStep: BaseStep = {
+      id: 'loopOuter',
+      action: 'foreach',
+      object_type: 'xpath',
+      object: "//div[@class='parent']",
+      index_key: 'i',
+      subSteps: [
+        {
+          id: 'meeting_title',
+          action: 'data',
+          object_type: 'xpath',
+          object: ".//h1",
+          key: 'meeting_title',
+          data_type: 'text'
+        },
+        {
+          id: 'loopInner',
+          action: 'foreach',
+          object_type: 'xpath',
+          object: ".//div[@class='child']",
+          index_key: 'j',
+          subSteps: [
+            {
+              id: 'attachment_name',
+              action: 'data',
+              object_type: 'xpath',
+              object: ".",
+              key: 'attachment_name',
+              data_type: 'text'
+            }
+          ]
+        }
+      ]
+    };
+
+    await executeStep(page, nestedStep, collector);
+
+    // Verify structure
+    // collector should have item_0 and item_1
+    expect(collector.item_0).toBeDefined();
+    expect(collector.item_1).toBeDefined();
+
+    // item_0 should have meeting_title and item_0, item_1 (inner items)
+    expect(collector.item_0.meeting_title).toBe('Meeting 1');
+    expect(collector.item_0.item_0.attachment_name).toBe('Attachment 1.1');
+    expect(collector.item_0.item_1.attachment_name).toBe('Attachment 1.2');
+
+    // VERY IMPORTANT: Parent context (meeting_title) should be in inner items because we copy it now
+    expect(collector.item_0.item_0.meeting_title).toBe('Meeting 1');
+    expect(collector.item_0.item_1.meeting_title).toBe('Meeting 1');
+
+    // Test flattening (implicitly happens in executeStep if we check final collector, 
+    // but executeStep doesn't flatten the ROOT collector, it only flattens on emit or if we call it)
+    
+    // items should be available for Meeting 2 as well
+    expect(collector.item_1.meeting_title).toBe('Meeting 2');
+    expect(collector.item_1.item_0.attachment_name).toBe('Attachment 2.1');
+    expect(collector.item_1.item_0.meeting_title).toBe('Meeting 2');
+  });
+
+  it('should correctly handle unique index placeholders {{i}} and {{j}}', async () => {
+    await navigate(page, testPageUrl);
+
+    const collector: Record<string, any> = {};
+    const nestedStep: BaseStep = {
+      id: 'loopOuter',
+      action: 'foreach',
+      object_type: 'xpath',
+      object: "//div[@class='parent']",
+      index_key: 'i',
+      subSteps: [
+        {
+          id: 'loopInner',
+          action: 'foreach',
+          object_type: 'xpath',
+          object: ".//div[@class='child']",
+          index_key: 'j',
+          subSteps: [
+            {
+              id: 'combined',
+              action: 'data',
+              object_type: 'xpath',
+              // Use both indices in a locator (hypothetical)
+              object: "//div[@id='p{{i}}c{{j}}']",
+              key: 'val',
+              data_type: 'text'
+            }
+          ]
+        }
+      ]
+    };
+
+    // We can't easily run this because the elements don't exist, 
+    // but we can verify that the sub-steps are cloned with correct replacements.
+    // Instead of executeStep, let's manually test cloneStepWithIndex.
+    
+    const { cloneStepWithIndex } = await import('../src/utils');
+    
+    // Outer loop idx 0
+    const outerCloned = cloneStepWithIndex(nestedStep.subSteps![0], 0, 'i');
+    
+    // Now outerCloned is the inner loop. Its object should NOT have i replaced (it doesn't use it)
+    // But its subSteps should have had {{i}} replaced if they used it.
+    const innerLoop = outerCloned;
+    const dataStep = innerLoop.subSteps![0];
+    
+    expect(dataStep.object).toBe("//div[@id='p0c{{j}}']");
+    
+    // Inner loop idx 5
+    const innerCloned = cloneStepWithIndex(dataStep, 5, 'j');
+    expect(innerCloned.object).toBe("//div[@id='p0c5']");
+  });
+});

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "stepwright",`
`3`		`- "version": "1.0.8",`
	`3`	`+ "version": "1.0.9",`
`4`	`4`	`"description": "A powerful web scraping library built with Playwright",`
`5`	`5`	`"main": "dist/index.js",`
`6`	`6`	`"types": "dist/index.d.ts",`
Original file line number	Diff line number	Diff line change
`@@ -19,6 +19,7 @@ export interface BaseStep {`
`19`	`19`	`terminateonerror?: boolean;`
`20`	`20`	`subSteps?: BaseStep[]; // for foreach/open`
`21`	`21`	`autoScroll?: boolean; // for foreach action - controls automatic scrolling (default: true)`
	`22`	`+ index_key?: string; // custom index placeholder for loops (e.g., 'j', 'k')`
`22`	`23`	`}`
`23`	`24`
`24`	`25`	`export interface PaginationConfig {`