Skip to content

Commit 911dc95

Browse files
mrkrslmrkrsl
committed
Add logging to observe Bing behaviour
1 parent 149e6c9 commit 911dc95

File tree

1 file changed

+135
-19
lines changed

1 file changed

+135
-19
lines changed

‎src/search-engine.ts‎

Lines changed: 135 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -214,14 +214,18 @@ export class SearchEngine {
214214
}
215215

216216
private async tryBrowserBingSearch(query: string, numResults: number, timeout: number): Promise<SearchResult[]> {
217-
console.log(`[SearchEngine] Trying browser-based Bing search with dedicated browser...`);
217+
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
218+
console.error(`[SearchEngine] BING: Starting browser-based search with dedicated browser for query: "${query}"`);
218219

219220
// Try with retry mechanism
220221
for (let attempt = 1; attempt <= 2; attempt++) {
221222
let browser;
222223
try {
224+
console.error(`[SearchEngine] BING: Attempt ${attempt}/2 - Launching Chromium browser...`);
225+
223226
// Create a dedicated browser instance for Bing search only
224227
const { chromium } = await import('playwright');
228+
const startTime = Date.now();
225229
browser = await chromium.launch({
226230
headless: process.env.BROWSER_HEADLESS !== 'false',
227231
args: [
@@ -232,23 +236,37 @@ export class SearchEngine {
232236
],
233237
});
234238

235-
console.log(`[SearchEngine] Bing search attempt ${attempt}/2 with fresh browser`);
239+
const launchTime = Date.now() - startTime;
240+
console.error(`[SearchEngine] BING: Browser launched successfully in ${launchTime}ms, connected: ${browser.isConnected()}`);
241+
236242
const results = await this.tryBrowserBingSearchInternal(browser, query, numResults, timeout);
243+
console.error(`[SearchEngine] BING: Search completed successfully with ${results.length} results`);
237244
return results;
238245
} catch (error) {
239-
console.error(`[SearchEngine] Bing search attempt ${attempt}/2 failed:`, error);
246+
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
247+
console.error(`[SearchEngine] BING: Attempt ${attempt}/2 FAILED with error: ${errorMessage}`);
248+
249+
if (debugBing) {
250+
console.error(`[SearchEngine] BING: Full error details:`, error);
251+
}
252+
240253
if (attempt === 2) {
254+
console.error(`[SearchEngine] BING: All attempts exhausted, giving up`);
241255
throw error; // Re-throw on final attempt
242256
}
243257
// Small delay before retry
258+
console.error(`[SearchEngine] BING: Waiting 500ms before retry...`);
244259
await new Promise(resolve => setTimeout(resolve, 500));
245260
} finally {
246261
// Always close the dedicated browser
247262
if (browser) {
248263
try {
249264
await browser.close();
265+
if (debugBing) {
266+
console.error(`[SearchEngine] BING: Browser closed successfully`);
267+
}
250268
} catch (closeError) {
251-
console.log(`[SearchEngine] Error closing Bing browser:`, closeError);
269+
console.error(`[SearchEngine] BING: Error closing browser:`, closeError);
252270
}
253271
}
254272
}
@@ -258,11 +276,16 @@ export class SearchEngine {
258276
}
259277

260278
private async tryBrowserBingSearchInternal(browser: any, query: string, numResults: number, timeout: number): Promise<SearchResult[]> {
279+
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
280+
261281
// Validate browser is still functional before proceeding
262282
if (!browser.isConnected()) {
283+
console.error(`[SearchEngine] BING: Browser is not connected`);
263284
throw new Error('Browser is not connected');
264285
}
265286

287+
console.error(`[SearchEngine] BING: Creating browser context with enhanced fingerprinting...`);
288+
266289
try {
267290
// Enhanced browser context with more realistic fingerprinting
268291
const context = await browser.newContext({
@@ -286,104 +309,181 @@ export class SearchEngine {
286309
}
287310
});
288311

312+
console.error(`[SearchEngine] BING: Context created, opening new page...`);
289313
const page = await context.newPage();
314+
console.error(`[SearchEngine] BING: Page opened successfully`);
290315

291316
try {
292317
// Try enhanced Bing search with proper web interface flow
293318
try {
319+
console.error(`[SearchEngine] BING: Attempting enhanced search (homepage → form submission)...`);
294320
const results = await this.tryEnhancedBingSearch(page, query, numResults, timeout);
321+
console.error(`[SearchEngine] BING: Enhanced search succeeded with ${results.length} results`);
295322
await context.close();
296323
return results;
297324
} catch (enhancedError) {
298-
console.log(`[SearchEngine] Enhanced Bing search failed, trying fallback: ${enhancedError instanceof Error ? enhancedError.message : 'Unknown error'}`);
325+
const errorMessage = enhancedError instanceof Error ? enhancedError.message : 'Unknown error';
326+
console.error(`[SearchEngine] BING: Enhanced search failed: ${errorMessage}`);
327+
328+
if (debugBing) {
329+
console.error(`[SearchEngine] BING: Enhanced search error details:`, enhancedError);
330+
}
331+
332+
console.error(`[SearchEngine] BING: Falling back to direct URL search...`);
299333

300334
// Fallback to direct URL approach with enhanced parameters
301335
const results = await this.tryDirectBingSearch(page, query, numResults, timeout);
336+
console.error(`[SearchEngine] BING: Direct search succeeded with ${results.length} results`);
302337
await context.close();
303338
return results;
304339
}
305340
} catch (error) {
306341
// Ensure context is closed even on error
342+
console.error(`[SearchEngine] BING: All search methods failed, closing context...`);
307343
await context.close();
308344
throw error;
309345
}
310346
} catch (error) {
311-
console.error(`[SearchEngine] Browser Bing search failed:`, error);
347+
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
348+
console.error(`[SearchEngine] BING: Internal search failed: ${errorMessage}`);
349+
350+
if (debugBing) {
351+
console.error(`[SearchEngine] BING: Internal search error details:`, error);
352+
}
353+
312354
throw error;
313355
}
314356
}
315357

316358
private async tryEnhancedBingSearch(page: any, query: string, numResults: number, timeout: number): Promise<SearchResult[]> {
317-
console.log(`[SearchEngine] Trying enhanced Bing search via web interface...`);
359+
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
360+
console.error(`[SearchEngine] BING: Enhanced search - navigating to Bing homepage...`);
318361

319362
// Navigate to Bing homepage first to establish proper session
363+
const startTime = Date.now();
320364
await page.goto('https://www.bing.com', {
321365
waitUntil: 'domcontentloaded',
322366
timeout: timeout / 2
323367
});
324368

369+
const loadTime = Date.now() - startTime;
370+
const pageTitle = await page.title();
371+
const currentUrl = page.url();
372+
console.error(`[SearchEngine] BING: Homepage loaded in ${loadTime}ms, title: "${pageTitle}", URL: ${currentUrl}`);
373+
325374
// Wait a moment for page to fully load
326375
await page.waitForTimeout(500);
327376

328377
// Find and use the search box (more realistic than direct URL)
329378
try {
379+
console.error(`[SearchEngine] BING: Looking for search form elements...`);
330380
await page.waitForSelector('#sb_form_q', { timeout: 2000 });
381+
console.error(`[SearchEngine] BING: Search box found, filling with query: "${query}"`);
331382
await page.fill('#sb_form_q', query);
332383

384+
console.error(`[SearchEngine] BING: Clicking search button and waiting for navigation...`);
333385
// Submit the search form
334386
await Promise.all([
335387
page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: timeout }),
336388
page.click('#search_icon')
337389
]);
338390

391+
const searchLoadTime = Date.now() - startTime;
392+
const searchPageTitle = await page.title();
393+
const searchPageUrl = page.url();
394+
console.error(`[SearchEngine] BING: Search completed in ${searchLoadTime}ms total, title: "${searchPageTitle}", URL: ${searchPageUrl}`);
395+
339396
} catch (formError) {
340-
console.log(`[SearchEngine] Search form submission failed, falling back to URL navigation`);
397+
const errorMessage = formError instanceof Error ? formError.message : 'Unknown error';
398+
console.error(`[SearchEngine] BING: Search form submission failed: ${errorMessage}`);
399+
400+
if (debugBing) {
401+
console.error(`[SearchEngine] BING: Form error details:`, formError);
402+
}
403+
341404
throw formError;
342405
}
343406

344407
// Wait for search results to load
345408
try {
409+
console.error(`[SearchEngine] BING: Waiting for search results to appear...`);
346410
await page.waitForSelector('.b_algo, .b_result', { timeout: 3000 });
411+
console.error(`[SearchEngine] BING: Search results selector found`);
347412
} catch {
348-
console.log(`[SearchEngine] Enhanced Bing results selector not found, proceeding anyway`);
413+
console.error(`[SearchEngine] BING: Search results selector not found, proceeding with page content anyway`);
349414
}
350415

351416
const html = await page.content();
352-
console.log(`[SearchEngine] Enhanced Bing got HTML with length: ${html.length}`);
417+
console.error(`[SearchEngine] BING: Got page HTML with length: ${html.length} characters`);
418+
419+
if (debugBing && html.length < 10000) {
420+
console.error(`[SearchEngine] BING: WARNING - HTML seems short, possible bot detection or error page`);
421+
}
353422

354423
const results = this.parseBingResults(html, numResults);
355-
console.log(`[SearchEngine] Enhanced Bing parsed ${results.length} results`);
424+
console.error(`[SearchEngine] BING: Enhanced search parsed ${results.length} results`);
425+
426+
if (results.length === 0) {
427+
console.error(`[SearchEngine] BING: WARNING - No results found, possible parsing failure or empty search`);
428+
429+
if (debugBing) {
430+
const sampleHtml = html.substring(0, 1000);
431+
console.error(`[SearchEngine] BING: Sample HTML for debugging:`, sampleHtml);
432+
}
433+
}
356434

357435
return results;
358436
}
359437

360438
private async tryDirectBingSearch(page: any, query: string, numResults: number, timeout: number): Promise<SearchResult[]> {
361-
console.log(`[SearchEngine] Trying direct Bing search with enhanced parameters...`);
439+
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
440+
console.error(`[SearchEngine] BING: Direct search with enhanced parameters...`);
362441

363442
// Generate a conversation ID (cvid) similar to what Bing uses
364443
const cvid = this.generateConversationId();
365444

366445
// Construct URL with enhanced parameters based on successful manual searches
367446
const searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&count=${Math.min(numResults, 10)}&form=QBLH&sp=-1&qs=n&cvid=${cvid}`;
368-
console.log(`[SearchEngine] Browser navigating to enhanced Bing URL: ${searchUrl}`);
447+
console.error(`[SearchEngine] BING: Navigating to direct URL: ${searchUrl}`);
369448

449+
const startTime = Date.now();
370450
await page.goto(searchUrl, {
371451
waitUntil: 'domcontentloaded',
372452
timeout: timeout
373453
});
454+
455+
const loadTime = Date.now() - startTime;
456+
const pageTitle = await page.title();
457+
const currentUrl = page.url();
458+
console.error(`[SearchEngine] BING: Direct page loaded in ${loadTime}ms, title: "${pageTitle}", URL: ${currentUrl}`);
374459

375460
// Wait for search results to load
376461
try {
462+
console.error(`[SearchEngine] BING: Waiting for search results to appear...`);
377463
await page.waitForSelector('.b_algo, .b_result', { timeout: 3000 });
464+
console.error(`[SearchEngine] BING: Search results selector found`);
378465
} catch {
379-
console.log(`[SearchEngine] Direct Bing results selector not found, proceeding anyway`);
466+
console.error(`[SearchEngine] BING: Search results selector not found, proceeding with page content anyway`);
380467
}
381468

382469
const html = await page.content();
383-
console.log(`[SearchEngine] Direct Bing got HTML with length: ${html.length}`);
470+
console.error(`[SearchEngine] BING: Got page HTML with length: ${html.length} characters`);
471+
472+
if (debugBing && html.length < 10000) {
473+
console.error(`[SearchEngine] BING: WARNING - HTML seems short, possible bot detection or error page`);
474+
}
384475

385476
const results = this.parseBingResults(html, numResults);
386-
console.log(`[SearchEngine] Direct Bing parsed ${results.length} results`);
477+
console.error(`[SearchEngine] BING: Direct search parsed ${results.length} results`);
478+
479+
if (results.length === 0) {
480+
console.error(`[SearchEngine] BING: WARNING - No results found, possible parsing failure or empty search`);
481+
482+
if (debugBing) {
483+
const sampleHtml = html.substring(0, 1000);
484+
console.error(`[SearchEngine] BING: Sample HTML for debugging:`, sampleHtml);
485+
}
486+
}
387487

388488
return results;
389489
}
@@ -679,27 +779,43 @@ export class SearchEngine {
679779
}
680780

681781
private parseBingResults(html: string, maxResults: number): SearchResult[] {
682-
console.log(`[SearchEngine] Parsing Bing HTML with length: ${html.length}`);
782+
const debugBing = process.env.DEBUG_BING_SEARCH === 'true';
783+
console.error(`[SearchEngine] BING: Parsing HTML with length: ${html.length}`);
683784

684785
const $ = cheerio.load(html);
685786
const results: SearchResult[] = [];
686787
const timestamp = generateTimestamp();
687788

789+
// Check for common Bing error indicators
790+
const pageTitle = $('title').text();
791+
console.error(`[SearchEngine] BING: Page title: "${pageTitle}"`);
792+
793+
if (pageTitle.includes('Access Denied') || pageTitle.includes('blocked') || pageTitle.includes('captcha')) {
794+
console.error(`[SearchEngine] BING: ERROR - Bot detection or access denied detected in page title`);
795+
}
796+
688797
// Bing result selectors
689798
const resultSelectors = [
690799
'.b_algo', // Main Bing results
691800
'.b_result', // Alternative Bing format
692801
'.b_card' // Card format
693802
];
694803

804+
console.error(`[SearchEngine] BING: Checking for result elements...`);
805+
806+
// Log counts for all selectors first
807+
for (const selector of resultSelectors) {
808+
const elements = $(selector);
809+
console.error(`[SearchEngine] BING: Found ${elements.length} elements with selector "${selector}"`);
810+
}
811+
695812
let foundResults = false;
696813

697814
for (const selector of resultSelectors) {
698815
if (foundResults && results.length >= maxResults) break;
699816

700-
console.log(`[SearchEngine] Trying Bing selector: ${selector}`);
701817
const elements = $(selector);
702-
console.log(`[SearchEngine] Found ${elements.length} elements with selector ${selector}`);
818+
if (elements.length === 0) continue;
703819

704820
elements.each((_index, element) => {
705821
if (results.length >= maxResults) return false;

0 commit comments

Comments
 (0)