diff --git a/scrapegraph-js/examples/mock_mode_example.js b/scrapegraph-js/examples/mock_mode_example.js new file mode 100644 index 0000000..492db4d --- /dev/null +++ b/scrapegraph-js/examples/mock_mode_example.js @@ -0,0 +1,297 @@ +/** + * Example demonstrating how to use the ScrapeGraph AI SDK in mock mode. + * + * This example shows how to: + * 1. Enable mock mode globally or per-request + * 2. Use custom mock responses + * 3. Use custom mock handlers + * 4. Test different endpoints in mock mode + * 5. Demonstrate environment variable activation + * + * Requirements: + * - Node.js 16+ + * - scrapegraph-js + * + * Usage: + * node mock_mode_example.js + * + * Or with environment variable: + * SGAI_MOCK=1 node mock_mode_example.js + */ + +import { + scrape, + getScrapeRequest, + smartScraper, + getSmartScraperRequest, + searchScraper, + getSearchScraperRequest, + markdownify, + getMarkdownifyRequest, + crawl, + getCrawlRequest, + agenticScraper, + getAgenticScraperRequest, + getCredits, + submitFeedback +} from '../index.js'; + +import { + initMockConfig, + enableMock, + disableMock, + setMockResponses, + setMockHandler +} from '../src/utils/mockConfig.js'; + +// Configuration +const API_KEY = process.env.SGAI_API_KEY || 'sgai-00000000-0000-0000-0000-000000000000'; + +/** + * Basic mock mode usage demonstration + */ +async function basicMockUsage() { + console.log('\n=== Basic Mock Usage ==='); + + // Enable mock mode globally + enableMock(); + + try { + // Test scrape endpoint + console.log('\n-- Testing scrape endpoint --'); + const scrapeResult = await scrape(API_KEY, 'https://example.com', { renderHeavyJs: true }); + console.log('Scrape result:', scrapeResult); + + // Test getScrapeRequest endpoint + console.log('\n-- Testing getScrapeRequest endpoint --'); + const scrapeStatus = await getScrapeRequest(API_KEY, 'mock-request-id'); + console.log('Scrape status:', scrapeStatus); + + // Test smartScraper endpoint + console.log('\n-- Testing smartScraper endpoint --'); + const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Extract the title'); + console.log('SmartScraper result:', smartResult); + + // Test getCredits endpoint + console.log('\n-- Testing getCredits endpoint --'); + const credits = await getCredits(API_KEY); + console.log('Credits:', credits); + + // Test submitFeedback endpoint + console.log('\n-- Testing submitFeedback endpoint --'); + const feedback = await submitFeedback(API_KEY, 'mock-request-id', 5, 'Great service!'); + console.log('Feedback result:', feedback); + + } catch (error) { + console.error('Error in basic mock usage:', error.message); + } +} + +/** + * Mock mode with custom responses + */ +async function mockWithCustomResponses() { + console.log('\n=== Mock Mode with Custom Responses ==='); + + // Set custom responses for specific endpoints + setMockResponses({ + '/v1/credits': { + remaining_credits: 42, + total_credits_used: 58, + custom_field: 'This is a custom response' + }, + '/v1/smartscraper': () => ({ + request_id: 'custom-mock-request-id', + custom_data: 'Generated by custom function' + }) + }); + + try { + // Test credits with custom response + console.log('\n-- Testing credits with custom response --'); + const credits = await getCredits(API_KEY); + console.log('Custom credits:', credits); + + // Test smartScraper with custom response + console.log('\n-- Testing smartScraper with custom response --'); + const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Extract data'); + console.log('Custom smartScraper result:', smartResult); + + } catch (error) { + console.error('Error in custom responses:', error.message); + } +} + +/** + * Mock mode with custom handler + */ +async function mockWithCustomHandler() { + console.log('\n=== Mock Mode with Custom Handler ==='); + + // Set a custom handler that overrides all responses + setMockHandler((method, url) => { + return { + custom_handler: true, + method: method, + url: url, + timestamp: new Date().toISOString(), + message: 'This response was generated by a custom handler' + }; + }); + + try { + // Test various endpoints with custom handler + console.log('\n-- Testing with custom handler --'); + + const scrapeResult = await scrape(API_KEY, 'https://example.com'); + console.log('Scrape with custom handler:', scrapeResult); + + const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Test prompt'); + console.log('SmartScraper with custom handler:', smartResult); + + const credits = await getCredits(API_KEY); + console.log('Credits with custom handler:', credits); + + } catch (error) { + console.error('Error in custom handler:', error.message); + } +} + +/** + * Per-request mock mode (without global enable) + */ +async function perRequestMockMode() { + console.log('\n=== Per-Request Mock Mode ==='); + + // Disable global mock mode + disableMock(); + + try { + // Test individual requests with mock enabled + console.log('\n-- Testing per-request mock mode --'); + + const scrapeResult = await scrape(API_KEY, 'https://example.com', { mock: true }); + console.log('Per-request mock scrape:', scrapeResult); + + const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Test', null, null, null, null, { mock: true }); + console.log('Per-request mock smartScraper:', smartResult); + + const scrapeStatus = await getScrapeRequest(API_KEY, 'test-id', { mock: true }); + console.log('Per-request mock getScrapeRequest:', scrapeStatus); + + } catch (error) { + console.error('Error in per-request mock mode:', error.message); + } +} + +/** + * Test all available endpoints in mock mode + */ +async function testAllEndpoints() { + console.log('\n=== Testing All Endpoints in Mock Mode ==='); + + enableMock(); + + try { + // Test all available endpoints + console.log('\n-- Testing all endpoints --'); + + // Scrape endpoints + const scrapeResult = await scrape(API_KEY, 'https://example.com'); + console.log('Scrape:', scrapeResult.request_id ? '✅' : '❌'); + + const scrapeStatus = await getScrapeRequest(API_KEY, 'mock-id'); + console.log('GetScrapeRequest:', scrapeStatus.status ? '✅' : '❌'); + + // SmartScraper endpoints + const smartResult = await smartScraper(API_KEY, 'https://example.com', 'Extract title'); + console.log('SmartScraper:', smartResult.request_id ? '✅' : '❌'); + + const smartStatus = await getSmartScraperRequest(API_KEY, 'mock-id'); + console.log('GetSmartScraperRequest:', smartStatus.status ? '✅' : '❌'); + + // SearchScraper endpoints + const searchResult = await searchScraper(API_KEY, 'Search for information'); + console.log('SearchScraper:', searchResult.request_id ? '✅' : '❌'); + + const searchStatus = await getSearchScraperRequest(API_KEY, 'mock-id'); + console.log('GetSearchScraperRequest:', searchStatus.status ? '✅' : '❌'); + + // Markdownify endpoints + const markdownResult = await markdownify(API_KEY, 'https://example.com'); + console.log('Markdownify:', markdownResult.request_id ? '✅' : '❌'); + + const markdownStatus = await getMarkdownifyRequest(API_KEY, 'mock-id'); + console.log('GetMarkdownifyRequest:', markdownStatus.status ? '✅' : '❌'); + + // Crawl endpoints + const crawlResult = await crawl(API_KEY, 'https://example.com'); + console.log('Crawl:', crawlResult.crawl_id ? '✅' : '❌'); + + const crawlStatus = await getCrawlRequest(API_KEY, 'mock-id'); + console.log('GetCrawlRequest:', crawlStatus.status ? '✅' : '❌'); + + // AgenticScraper endpoints + const agenticResult = await agenticScraper(API_KEY, 'https://example.com', ['click button']); + console.log('AgenticScraper:', agenticResult.request_id ? '✅' : '❌'); + + const agenticStatus = await getAgenticScraperRequest(API_KEY, 'mock-id'); + console.log('GetAgenticScraperRequest:', agenticStatus.status ? '✅' : '❌'); + + // Utility endpoints + const credits = await getCredits(API_KEY); + console.log('GetCredits:', credits.remaining_credits ? '✅' : '❌'); + + const feedback = await submitFeedback(API_KEY, 'mock-id', 5, 'Great!'); + console.log('SubmitFeedback:', feedback.status ? '✅' : '❌'); + + } catch (error) { + console.error('Error testing endpoints:', error.message); + } +} + +/** + * Environment variable activation test + */ +async function testEnvironmentActivation() { + console.log('\n=== Environment Variable Activation Test ==='); + + console.log('Current SGAI_MOCK value:', process.env.SGAI_MOCK || 'not set'); + + // Reinitialize mock config to check environment + initMockConfig(); + + try { + const credits = await getCredits(API_KEY); + console.log('Credits with env check:', credits); + } catch (error) { + console.error('Error in environment test:', error.message); + } +} + +/** + * Main function to run all examples + */ +async function main() { + console.log('🧪 ScrapeGraph AI SDK - Mock Mode Examples'); + console.log('=========================================='); + + try { + await basicMockUsage(); + await mockWithCustomResponses(); + await mockWithCustomHandler(); + await perRequestMockMode(); + await testAllEndpoints(); + await testEnvironmentActivation(); + + console.log('\n✅ All mock mode examples completed successfully!'); + + } catch (error) { + console.error('\n❌ Error running examples:', error.message); + } +} + +// Run the examples +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/scrapegraph-js/index.js b/scrapegraph-js/index.js index e41f6d7..a29a3d7 100644 --- a/scrapegraph-js/index.js +++ b/scrapegraph-js/index.js @@ -6,3 +6,14 @@ export { searchScraper, getSearchScraperRequest } from './src/searchScraper.js'; export { getCredits } from './src/credits.js'; export { sendFeedback } from './src/feedback.js'; export { crawl, getCrawlRequest } from './src/crawl.js'; + +// Mock utilities +export { + initMockConfig, + enableMock, + disableMock, + setMockResponses, + setMockHandler, + getMockConfig, + isMockEnabled +} from './src/utils/mockConfig.js'; diff --git a/scrapegraph-js/src/credits.js b/scrapegraph-js/src/credits.js index d6c5465..a6c5814 100644 --- a/scrapegraph-js/src/credits.js +++ b/scrapegraph-js/src/credits.js @@ -1,5 +1,7 @@ import axios from 'axios'; import handleError from './utils/handleError.js'; +import { isMockEnabled, getMockConfig } from './utils/mockConfig.js'; +import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.js'; /** * Retrieve credits from the API. @@ -7,7 +9,19 @@ import handleError from './utils/handleError.js'; * @param {string} apiKey - Your ScrapeGraph AI API key * @returns {Promise} Response from the API in JSON format */ -export async function getCredits(apiKey) { +export async function getCredits(apiKey, options = {}) { + const { mock = null } = options; + + // Check if mock mode is enabled + const useMock = mock !== null ? mock : isMockEnabled(); + + if (useMock) { + console.log('🧪 Mock mode active. Returning stub for getCredits'); + const mockConfig = getMockConfig(); + const mockData = getMockResponse('GET', 'https://api.scrapegraphai.com/v1/credits', mockConfig.customResponses, mockConfig.customHandler); + return mockData; + } + const endpoint = 'https://api.scrapegraphai.com/v1/credits'; const headers = { 'accept': 'application/json', diff --git a/scrapegraph-js/src/scrape.js b/scrapegraph-js/src/scrape.js index 09c0a9a..a6075e9 100644 --- a/scrapegraph-js/src/scrape.js +++ b/scrapegraph-js/src/scrape.js @@ -1,5 +1,7 @@ import axios from 'axios'; import handleError from './utils/handleError.js'; +import { isMockEnabled, getMockConfig } from './utils/mockConfig.js'; +import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.js'; /** * Converts a webpage into HTML format with optional JavaScript rendering. @@ -44,9 +46,20 @@ import handleError from './utils/handleError.js'; export async function scrape(apiKey, url, options = {}) { const { renderHeavyJs = false, - headers: customHeaders = {} + headers: customHeaders = {}, + mock = null } = options; + // Check if mock mode is enabled + const useMock = mock !== null ? mock : isMockEnabled(); + + if (useMock) { + console.log('🧪 Mock mode active. Returning stub for scrape request'); + const mockConfig = getMockConfig(); + const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/scrape', mockConfig.customResponses, mockConfig.customHandler); + return mockData; + } + const endpoint = 'https://api.scrapegraphai.com/v1/scrape'; const headers = { 'accept': 'application/json', @@ -114,7 +127,19 @@ export async function scrape(apiKey, url, options = {}) { * - CSS styles and formatting * - Images, links, and other media elements */ -export async function getScrapeRequest(apiKey, requestId) { +export async function getScrapeRequest(apiKey, requestId, options = {}) { + const { mock = null } = options; + + // Check if mock mode is enabled + const useMock = mock !== null ? mock : isMockEnabled(); + + if (useMock) { + console.log('🧪 Mock mode active. Returning stub for getScrapeRequest'); + const mockConfig = getMockConfig(); + const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/scrape/${requestId}`, mockConfig.customResponses, mockConfig.customHandler); + return mockData; + } + const endpoint = 'https://api.scrapegraphai.com/v1/scrape/' + requestId; const headers = { 'accept': 'application/json', diff --git a/scrapegraph-js/src/smartScraper.js b/scrapegraph-js/src/smartScraper.js index 80d4ed4..f8ef62a 100644 --- a/scrapegraph-js/src/smartScraper.js +++ b/scrapegraph-js/src/smartScraper.js @@ -2,6 +2,8 @@ import axios from 'axios'; import handleError from './utils/handleError.js'; import { ZodType } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; +import { isMockEnabled, getMockConfig } from './utils/mockConfig.js'; +import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.js'; /** * Scrape and extract structured data from a webpage using ScrapeGraph AI. @@ -16,7 +18,19 @@ import { zodToJsonSchema } from 'zod-to-json-schema'; * @returns {Promise} Extracted data in JSON format matching the provided schema * @throws - Will throw an error in case of an HTTP failure. */ -export async function smartScraper(apiKey, url, prompt, schema = null, numberOfScrolls = null, totalPages = null, cookies = null) { +export async function smartScraper(apiKey, url, prompt, schema = null, numberOfScrolls = null, totalPages = null, cookies = null, options = {}) { + const { mock = null } = options; + + // Check if mock mode is enabled + const useMock = mock !== null ? mock : isMockEnabled(); + + if (useMock) { + console.log('🧪 Mock mode active. Returning stub for smartScraper request'); + const mockConfig = getMockConfig(); + const mockData = getMockResponse('POST', 'https://api.scrapegraphai.com/v1/smartscraper', mockConfig.customResponses, mockConfig.customHandler); + return mockData; + } + const endpoint = 'https://api.scrapegraphai.com/v1/smartscraper'; const headers = { 'accept': 'application/json', @@ -98,7 +112,19 @@ export async function smartScraper(apiKey, url, prompt, schema = null, numberOfS * console.error('Error fetching request:', error); * } */ -export async function getSmartScraperRequest(apiKey, requestId) { +export async function getSmartScraperRequest(apiKey, requestId, options = {}) { + const { mock = null } = options; + + // Check if mock mode is enabled + const useMock = mock !== null ? mock : isMockEnabled(); + + if (useMock) { + console.log('🧪 Mock mode active. Returning stub for getSmartScraperRequest'); + const mockConfig = getMockConfig(); + const mockData = getMockResponse('GET', `https://api.scrapegraphai.com/v1/smartscraper/${requestId}`, mockConfig.customResponses, mockConfig.customHandler); + return mockData; + } + const endpoint = 'https://api.scrapegraphai.com/v1/smartscraper/' + requestId; const headers = { 'accept': 'application/json', diff --git a/scrapegraph-js/src/utils/mockConfig.js b/scrapegraph-js/src/utils/mockConfig.js new file mode 100644 index 0000000..3f10f77 --- /dev/null +++ b/scrapegraph-js/src/utils/mockConfig.js @@ -0,0 +1,100 @@ +/** + * Mock configuration utility for ScrapeGraph AI SDK + * Manages global mock settings and configuration + */ + +// Global mock configuration +let mockConfig = { + enabled: false, + customResponses: {}, + customHandler: null +}; + +/** + * Check if mock mode is enabled via environment variable + * @returns {boolean} True if mock mode should be enabled + */ +function isMockEnabledFromEnv() { + if (typeof process !== 'undefined' && process.env) { + const mockEnv = process.env.SGAI_MOCK; + if (mockEnv) { + return ['1', 'true', 'True', 'TRUE', 'yes', 'YES', 'on', 'ON'].includes(mockEnv.trim()); + } + } + return false; +} + +/** + * Initialize mock configuration + * @param {Object} options - Mock configuration options + * @param {boolean} options.enabled - Whether mock mode is enabled + * @param {Object} options.customResponses - Custom response overrides + * @param {Function} options.customHandler - Custom handler function + */ +export function initMockConfig(options = {}) { + const { + enabled = isMockEnabledFromEnv(), + customResponses = {}, + customHandler = null + } = options; + + mockConfig = { + enabled: Boolean(enabled), + customResponses: { ...customResponses }, + customHandler: customHandler + }; + + if (mockConfig.enabled) { + console.log('🧪 ScrapeGraph AI SDK: Mock mode enabled'); + } +} + +/** + * Get current mock configuration + * @returns {Object} Current mock configuration + */ +export function getMockConfig() { + return { ...mockConfig }; +} + +/** + * Check if mock mode is currently enabled + * @returns {boolean} True if mock mode is enabled + */ +export function isMockEnabled() { + return mockConfig.enabled; +} + +/** + * Set custom responses for specific endpoints + * @param {Object} responses - Map of endpoint paths to responses + */ +export function setMockResponses(responses) { + mockConfig.customResponses = { ...mockConfig.customResponses, ...responses }; +} + +/** + * Set a custom mock handler function + * @param {Function} handler - Custom handler function + */ +export function setMockHandler(handler) { + mockConfig.customHandler = handler; +} + +/** + * Disable mock mode + */ +export function disableMock() { + mockConfig.enabled = false; +} + +/** + * Enable mock mode + */ +export function enableMock() { + mockConfig.enabled = true; + console.log('🧪 ScrapeGraph AI SDK: Mock mode enabled'); +} + +// Initialize with environment check +initMockConfig(); diff --git a/scrapegraph-js/src/utils/mockResponse.js b/scrapegraph-js/src/utils/mockResponse.js new file mode 100644 index 0000000..4f4339d --- /dev/null +++ b/scrapegraph-js/src/utils/mockResponse.js @@ -0,0 +1,142 @@ +/** + * Mock response utility for ScrapeGraph AI SDK + * Provides deterministic mock responses when mock mode is enabled + */ + +/** + * Generate a mock UUID with a prefix + * @param {string} prefix - Prefix for the mock ID + * @returns {string} Mock UUID + */ +function generateMockId(prefix = 'mock') { + const timestamp = Date.now().toString(36); + const random = Math.random().toString(36).substring(2, 8); + return `${prefix}-${timestamp}-${random}`; +} + +/** + * Get mock response based on endpoint and method + * @param {string} method - HTTP method (GET, POST, etc.) + * @param {string} url - Full URL + * @param {Object} customResponses - Custom response overrides + * @param {Function} customHandler - Custom handler function + * @returns {Object} Mock response data + */ +export function getMockResponse(method, url, customResponses = {}, customHandler = null) { + // Custom handler takes precedence + if (customHandler && typeof customHandler === 'function') { + try { + return customHandler(method, url); + } catch (error) { + console.warn('Custom mock handler failed, falling back to defaults:', error.message); + } + } + + // Parse URL to get path + const urlObj = new URL(url); + const path = urlObj.pathname; + + // Check for custom response override + if (customResponses[path]) { + const override = customResponses[path]; + return typeof override === 'function' ? override() : override; + } + + const upperMethod = method.toUpperCase(); + + // Credits endpoint + if (path.endsWith('/credits') && upperMethod === 'GET') { + return { + remaining_credits: 1000, + total_credits_used: 0 + }; + } + + // Feedback endpoint + if (path.endsWith('/feedback') && upperMethod === 'POST') { + return { + status: 'success' + }; + } + + // Create-like endpoints (POST) + if (upperMethod === 'POST') { + if (path.endsWith('/crawl')) { + return { + crawl_id: generateMockId('mock-crawl') + }; + } + // All other POST endpoints return a request id + return { + request_id: generateMockId('mock-req') + }; + } + + // Status-like endpoints (GET) + if (upperMethod === 'GET') { + if (path.includes('markdownify')) { + return { + status: 'completed', + content: '# Mock markdown\n\nThis is a mock markdown response...' + }; + } + if (path.includes('smartscraper')) { + return { + status: 'completed', + result: [{ field: 'value', title: 'Mock Title' }] + }; + } + if (path.includes('searchscraper')) { + return { + status: 'completed', + results: [{ url: 'https://example.com', title: 'Mock Result' }] + }; + } + if (path.includes('crawl')) { + return { + status: 'completed', + pages: [] + }; + } + if (path.includes('agentic-scrapper')) { + return { + status: 'completed', + actions: [] + }; + } + if (path.includes('scrape')) { + return { + status: 'completed', + html: 'Mock HTML

Mock Content

' + }; + } + } + + // Generic fallback + return { + status: 'mock', + url: url, + method: method, + message: 'Mock response generated' + }; +} + +/** + * Create a mock axios response object + * @param {Object} data - Response data + * @returns {Object} Mock axios response + */ +export function createMockAxiosResponse(data) { + return { + data, + status: 200, + statusText: 'OK', + headers: { + 'content-type': 'application/json' + }, + config: { + url: 'mock-url', + method: 'mock' + } + }; +} diff --git a/scrapegraph-py/examples/async/mock/async_mock_mode_example.py b/scrapegraph-py/examples/async/mock/async_mock_mode_example.py new file mode 100644 index 0000000..31d3328 --- /dev/null +++ b/scrapegraph-py/examples/async/mock/async_mock_mode_example.py @@ -0,0 +1,61 @@ +import asyncio + +from scrapegraph_py import AsyncClient +from scrapegraph_py.logger import sgai_logger + + +sgai_logger.set_logging(level="INFO") + + +async def basic_mock_usage(): + # Initialize the client with mock mode enabled + async with AsyncClient.from_env(mock=True) as client: + print("\n-- get_credits (mock) --") + print(await client.get_credits()) + + print("\n-- markdownify (mock) --") + md = await client.markdownify(website_url="https://example.com") + print(md) + + print("\n-- get_markdownify (mock) --") + md_status = await client.get_markdownify("00000000-0000-0000-0000-000000000123") + print(md_status) + + print("\n-- smartscraper (mock) --") + ss = await client.smartscraper(user_prompt="Extract title", website_url="https://example.com") + print(ss) + + +async def mock_with_path_overrides(): + # Initialize the client with mock mode and custom responses + async with AsyncClient.from_env( + mock=True, + mock_responses={ + "/v1/credits": {"remaining_credits": 42, "total_credits_used": 58} + }, + ) as client: + print("\n-- get_credits with override (mock) --") + print(await client.get_credits()) + + +async def mock_with_custom_handler(): + def handler(method, url, kwargs): + return {"handled_by": "custom_handler", "method": method, "url": url} + + # Initialize the client with mock mode and custom handler + async with AsyncClient.from_env(mock=True, mock_handler=handler) as client: + print("\n-- searchscraper via custom handler (mock) --") + resp = await client.searchscraper(user_prompt="Search something") + print(resp) + + +async def main(): + await basic_mock_usage() + await mock_with_path_overrides() + await mock_with_custom_handler() + + +if __name__ == "__main__": + asyncio.run(main()) + + diff --git a/scrapegraph-py/examples/sync/mock/mock_mode_example.py b/scrapegraph-py/examples/sync/mock/mock_mode_example.py new file mode 100644 index 0000000..c2bc8b1 --- /dev/null +++ b/scrapegraph-py/examples/sync/mock/mock_mode_example.py @@ -0,0 +1,58 @@ +from scrapegraph_py import Client +from scrapegraph_py.logger import sgai_logger + + +sgai_logger.set_logging(level="INFO") + + +def basic_mock_usage(): + # Initialize the client with mock mode enabled + client = Client.from_env(mock=True) + + print("\n-- get_credits (mock) --") + print(client.get_credits()) + + print("\n-- markdownify (mock) --") + md = client.markdownify(website_url="https://example.com") + print(md) + + print("\n-- get_markdownify (mock) --") + md_status = client.get_markdownify("00000000-0000-0000-0000-000000000123") + print(md_status) + + print("\n-- smartscraper (mock) --") + ss = client.smartscraper(user_prompt="Extract title", website_url="https://example.com") + print(ss) + + +def mock_with_path_overrides(): + # Initialize the client with mock mode and custom responses + client = Client.from_env( + mock=True, + mock_responses={ + "/v1/credits": {"remaining_credits": 42, "total_credits_used": 58} + }, + ) + + print("\n-- get_credits with override (mock) --") + print(client.get_credits()) + + +def mock_with_custom_handler(): + def handler(method, url, kwargs): + return {"handled_by": "custom_handler", "method": method, "url": url} + + # Initialize the client with mock mode and custom handler + client = Client.from_env(mock=True, mock_handler=handler) + + print("\n-- searchscraper via custom handler (mock) --") + resp = client.searchscraper(user_prompt="Search something") + print(resp) + + +if __name__ == "__main__": + basic_mock_usage() + mock_with_path_overrides() + mock_with_custom_handler() + + diff --git a/scrapegraph-py/scrapegraph_py/async_client.py b/scrapegraph-py/scrapegraph_py/async_client.py index b965838..a24bd2c 100644 --- a/scrapegraph-py/scrapegraph_py/async_client.py +++ b/scrapegraph-py/scrapegraph_py/async_client.py @@ -1,9 +1,11 @@ import asyncio -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Callable from aiohttp import ClientSession, ClientTimeout, TCPConnector from aiohttp.client_exceptions import ClientError from pydantic import BaseModel +from urllib.parse import urlparse +import uuid as _uuid from scrapegraph_py.config import API_BASE_URL, DEFAULT_HEADERS from scrapegraph_py.exceptions import APIError @@ -35,6 +37,9 @@ def from_env( timeout: Optional[float] = None, max_retries: int = 3, retry_delay: float = 1.0, + mock: Optional[bool] = None, + mock_handler: Optional[Callable[[str, str, Dict[str, Any]], Any]] = None, + mock_responses: Optional[Dict[str, Any]] = None, ): """Initialize AsyncClient using API key from environment variable. @@ -46,15 +51,27 @@ def from_env( """ from os import getenv + # Allow enabling mock mode from environment if not explicitly provided + if mock is None: + mock_env = getenv("SGAI_MOCK", "0").strip().lower() + mock = mock_env in {"1", "true", "yes", "on"} + api_key = getenv("SGAI_API_KEY") + # In mock mode, we don't need a real API key if not api_key: - raise ValueError("SGAI_API_KEY environment variable not set") + if mock: + api_key = "sgai-00000000-0000-0000-0000-000000000000" + else: + raise ValueError("SGAI_API_KEY environment variable not set") return cls( api_key=api_key, verify_ssl=verify_ssl, timeout=timeout, max_retries=max_retries, retry_delay=retry_delay, + mock=bool(mock), + mock_handler=mock_handler, + mock_responses=mock_responses, ) def __init__( @@ -64,6 +81,9 @@ def __init__( timeout: Optional[float] = None, max_retries: int = 3, retry_delay: float = 1.0, + mock: bool = False, + mock_handler: Optional[Callable[[str, str, Dict[str, Any]], Any]] = None, + mock_responses: Optional[Dict[str, Any]] = None, ): """Initialize AsyncClient with configurable parameters. @@ -96,6 +116,9 @@ def __init__( self.headers = {**DEFAULT_HEADERS, "SGAI-APIKEY": api_key} self.max_retries = max_retries self.retry_delay = retry_delay + self.mock = bool(mock) + self.mock_handler = mock_handler + self.mock_responses = mock_responses or {} ssl = None if verify_ssl else False self.timeout = ClientTimeout(total=timeout) if timeout is not None else None @@ -108,6 +131,9 @@ def __init__( async def _make_request(self, method: str, url: str, **kwargs) -> Any: """Make HTTP request with retry logic.""" + # Short-circuit when mock mode is enabled + if getattr(self, "mock", False): + return self._mock_response(method, url, **kwargs) for attempt in range(self.max_retries): try: logger.info( @@ -145,6 +171,71 @@ async def _make_request(self, method: str, url: str, **kwargs) -> Any: logger.info(f"⏳ Waiting {retry_delay}s before retry {attempt + 2}") await asyncio.sleep(retry_delay) + def _mock_response(self, method: str, url: str, **kwargs) -> Any: + """Return a deterministic mock response without performing network I/O. + + Resolution order: + 1) If a custom mock_handler is provided, delegate to it + 2) If mock_responses contains a key for the request path, use it + 3) Fallback to built-in defaults per endpoint family + """ + logger.info(f"🧪 Mock mode active. Returning stub for {method} {url}") + + # 1) Custom handler + if self.mock_handler is not None: + try: + return self.mock_handler(method, url, kwargs) + except Exception as handler_error: + logger.warning(f"Custom mock_handler raised: {handler_error}. Falling back to defaults.") + + # 2) Path-based override + try: + parsed = urlparse(url) + path = parsed.path.rstrip("/") + except Exception: + path = url + + override = self.mock_responses.get(path) + if override is not None: + return override() if callable(override) else override + + # 3) Built-in defaults + def new_id(prefix: str) -> str: + return f"{prefix}-{_uuid.uuid4()}" + + upper_method = method.upper() + + # Credits endpoint + if path.endswith("/credits") and upper_method == "GET": + return {"remaining_credits": 1000, "total_credits_used": 0} + + # Feedback acknowledge + if path.endswith("/feedback") and upper_method == "POST": + return {"status": "success"} + + # Create-like endpoints (POST) + if upper_method == "POST": + if path.endswith("/crawl"): + return {"crawl_id": new_id("mock-crawl")} + # All other POST endpoints return a request id + return {"request_id": new_id("mock-req")} + + # Status-like endpoints (GET) + if upper_method == "GET": + if "markdownify" in path: + return {"status": "completed", "content": "# Mock markdown\n\n..."} + if "smartscraper" in path: + return {"status": "completed", "result": [{"field": "value"}]} + if "searchscraper" in path: + return {"status": "completed", "results": [{"url": "https://example.com"}]} + if "crawl" in path: + return {"status": "completed", "pages": []} + if "agentic-scrapper" in path: + return {"status": "completed", "actions": []} + + # Generic fallback + return {"status": "mock", "url": url, "method": method, "kwargs": kwargs} + async def markdownify( self, website_url: str, headers: Optional[dict[str, str]] = None ): diff --git a/scrapegraph-py/scrapegraph_py/client.py b/scrapegraph-py/scrapegraph_py/client.py index 6e1d37a..b024438 100644 --- a/scrapegraph-py/scrapegraph_py/client.py +++ b/scrapegraph-py/scrapegraph_py/client.py @@ -1,10 +1,12 @@ # Client implementation goes here -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Callable import requests import urllib3 from pydantic import BaseModel from requests.exceptions import RequestException +from urllib.parse import urlparse +import uuid as _uuid from scrapegraph_py.config import API_BASE_URL, DEFAULT_HEADERS from scrapegraph_py.exceptions import APIError @@ -36,6 +38,9 @@ def from_env( timeout: Optional[float] = None, max_retries: int = 3, retry_delay: float = 1.0, + mock: Optional[bool] = None, + mock_handler: Optional[Callable[[str, str, Dict[str, Any]], Any]] = None, + mock_responses: Optional[Dict[str, Any]] = None, ): """Initialize Client using API key from environment variable. @@ -44,18 +49,32 @@ def from_env( timeout: Request timeout in seconds. None means no timeout (infinite) max_retries: Maximum number of retry attempts retry_delay: Delay between retries in seconds + mock: If True, the client will not perform real HTTP requests and + will return stubbed responses. If None, reads from SGAI_MOCK env. """ from os import getenv + # Allow enabling mock mode from environment if not explicitly provided + if mock is None: + mock_env = getenv("SGAI_MOCK", "0").strip().lower() + mock = mock_env in {"1", "true", "yes", "on"} + api_key = getenv("SGAI_API_KEY") + # In mock mode, we don't need a real API key if not api_key: - raise ValueError("SGAI_API_KEY environment variable not set") + if mock: + api_key = "sgai-00000000-0000-0000-0000-000000000000" + else: + raise ValueError("SGAI_API_KEY environment variable not set") return cls( api_key=api_key, verify_ssl=verify_ssl, timeout=timeout, max_retries=max_retries, retry_delay=retry_delay, + mock=bool(mock), + mock_handler=mock_handler, + mock_responses=mock_responses, ) def __init__( @@ -65,6 +84,9 @@ def __init__( timeout: Optional[float] = None, max_retries: int = 3, retry_delay: float = 1.0, + mock: bool = False, + mock_handler: Optional[Callable[[str, str, Dict[str, Any]], Any]] = None, + mock_responses: Optional[Dict[str, Any]] = None, ): """Initialize Client with configurable parameters. @@ -75,6 +97,12 @@ def __init__( timeout: Request timeout in seconds. None means no timeout (infinite) max_retries: Maximum number of retry attempts retry_delay: Delay between retries in seconds + mock: If True, the client will bypass HTTP calls and return + deterministic mock responses + mock_handler: Optional callable to generate custom mock responses + given (method, url, request_kwargs) + mock_responses: Optional mapping of path (e.g. "/v1/credits") to + static response or callable returning a response """ logger.info("🔑 Initializing Client") @@ -99,6 +127,9 @@ def __init__( self.timeout = timeout self.max_retries = max_retries self.retry_delay = retry_delay + self.mock = bool(mock) + self.mock_handler = mock_handler + self.mock_responses = mock_responses or {} # Create a session for connection pooling self.session = requests.Session() @@ -124,6 +155,9 @@ def __init__( def _make_request(self, method: str, url: str, **kwargs) -> Any: """Make HTTP request with error handling.""" + # Short-circuit when mock mode is enabled + if getattr(self, "mock", False): + return self._mock_response(method, url, **kwargs) try: logger.info(f"🚀 Making {method} request to {url}") logger.debug(f"🔍 Request parameters: {kwargs}") @@ -156,6 +190,71 @@ def _make_request(self, method: str, url: str, **kwargs) -> Any: logger.error(f"🔴 Connection Error: {str(e)}") raise ConnectionError(f"Failed to connect to API: {str(e)}") + def _mock_response(self, method: str, url: str, **kwargs) -> Any: + """Return a deterministic mock response without performing network I/O. + + Resolution order: + 1) If a custom mock_handler is provided, delegate to it + 2) If mock_responses contains a key for the request path, use it + 3) Fallback to built-in defaults per endpoint family + """ + logger.info(f"🧪 Mock mode active. Returning stub for {method} {url}") + + # 1) Custom handler + if self.mock_handler is not None: + try: + return self.mock_handler(method, url, kwargs) + except Exception as handler_error: + logger.warning(f"Custom mock_handler raised: {handler_error}. Falling back to defaults.") + + # 2) Path-based override + try: + parsed = urlparse(url) + path = parsed.path.rstrip("/") + except Exception: + path = url + + override = self.mock_responses.get(path) + if override is not None: + return override() if callable(override) else override + + # 3) Built-in defaults + def new_id(prefix: str) -> str: + return f"{prefix}-{_uuid.uuid4()}" + + upper_method = method.upper() + + # Credits endpoint + if path.endswith("/credits") and upper_method == "GET": + return {"remaining_credits": 1000, "total_credits_used": 0} + + # Feedback acknowledge + if path.endswith("/feedback") and upper_method == "POST": + return {"status": "success"} + + # Create-like endpoints (POST) + if upper_method == "POST": + if path.endswith("/crawl"): + return {"crawl_id": new_id("mock-crawl")} + # All other POST endpoints return a request id + return {"request_id": new_id("mock-req")} + + # Status-like endpoints (GET) + if upper_method == "GET": + if "markdownify" in path: + return {"status": "completed", "content": "# Mock markdown\n\n..."} + if "smartscraper" in path: + return {"status": "completed", "result": [{"field": "value"}]} + if "searchscraper" in path: + return {"status": "completed", "results": [{"url": "https://example.com"}]} + if "crawl" in path: + return {"status": "completed", "pages": []} + if "agentic-scrapper" in path: + return {"status": "completed", "actions": []} + + # Generic fallback + return {"status": "mock", "url": url, "method": method, "kwargs": kwargs} + def markdownify(self, website_url: str, headers: Optional[dict[str, str]] = None): """Send a markdownify request""" logger.info(f"🔍 Starting markdownify request for {website_url}") diff --git a/scrapegraph-py/tests/test_mock_async_client.py b/scrapegraph-py/tests/test_mock_async_client.py new file mode 100644 index 0000000..054f029 --- /dev/null +++ b/scrapegraph-py/tests/test_mock_async_client.py @@ -0,0 +1,345 @@ +""" +Tests for AsyncClient mock mode functionality +""" +import os +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from pydantic import BaseModel + +from scrapegraph_py.async_client import AsyncClient +from tests.utils import generate_mock_api_key + + +@pytest.fixture +def mock_api_key(): + return generate_mock_api_key() + + +@pytest.fixture +def mock_uuid(): + return str(uuid4()) + + +class TestAsyncMockMode: + """Test basic async mock mode functionality""" + + @pytest.mark.asyncio + async def test_async_client_mock_mode_basic(self, mock_api_key): + """Test that async mock mode bypasses HTTP calls and returns stub data""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # Test credits endpoint + credits = await client.get_credits() + assert credits["remaining_credits"] == 1000 + assert credits["total_credits_used"] == 0 + + # Test smartscraper POST endpoint + response = await client.smartscraper( + user_prompt="Extract title", + website_url="https://example.com" + ) + assert "request_id" in response + assert response["request_id"].startswith("mock-req-") + + # Test feedback endpoint + feedback = await client.submit_feedback("test-id", 5, "Great!") + assert feedback["status"] == "success" + + @pytest.mark.asyncio + async def test_async_client_mock_mode_get_endpoints(self, mock_api_key, mock_uuid): + """Test GET endpoints in async mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # Test markdownify GET + md_result = await client.get_markdownify(mock_uuid) + assert md_result["status"] == "completed" + assert "Mock markdown" in md_result["content"] + + # Test smartscraper GET + ss_result = await client.get_smartscraper(mock_uuid) + assert ss_result["status"] == "completed" + assert "result" in ss_result + + # Test searchscraper GET + search_result = await client.get_searchscraper(mock_uuid) + assert search_result["status"] == "completed" + assert "results" in search_result + + @pytest.mark.asyncio + async def test_async_client_mock_mode_crawl_endpoints(self, mock_api_key, mock_uuid): + """Test crawl-specific endpoints in async mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # Test crawl POST + crawl_response = await client.crawl(url="https://example.com") + assert "crawl_id" in crawl_response + assert crawl_response["crawl_id"].startswith("mock-crawl-") + + # Test crawl GET + crawl_result = await client.get_crawl(mock_uuid) + assert crawl_result["status"] == "completed" + assert "pages" in crawl_result + + @pytest.mark.asyncio + async def test_async_client_mock_mode_agentic_scraper(self, mock_api_key, mock_uuid): + """Test agentic scraper endpoints in async mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # Test agentic scraper POST + response = await client.agenticscraper( + url="https://example.com", + steps=["click button", "extract data"] + ) + assert "request_id" in response + assert response["request_id"].startswith("mock-req-") + + # Test agentic scraper GET + result = await client.get_agenticscraper(mock_uuid) + assert result["status"] == "completed" + assert "actions" in result + + @pytest.mark.asyncio + async def test_async_client_mock_mode_searchscraper(self, mock_api_key): + """Test searchscraper endpoint in async mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + response = await client.searchscraper( + user_prompt="Search for information", + num_results=5 + ) + assert "request_id" in response + assert response["request_id"].startswith("mock-req-") + + @pytest.mark.asyncio + async def test_async_client_mock_mode_markdownify(self, mock_api_key): + """Test markdownify endpoint in async mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + response = await client.markdownify( + website_url="https://example.com", + headers={"User-Agent": "test"} + ) + assert "request_id" in response + assert response["request_id"].startswith("mock-req-") + + +class TestAsyncMockModeCustomization: + """Test async mock mode customization features""" + + @pytest.mark.asyncio + async def test_async_client_mock_responses_override(self, mock_api_key): + """Test custom mock responses via mock_responses parameter""" + custom_responses = { + "/v1/credits": {"remaining_credits": 42, "total_credits_used": 58} + } + + async with AsyncClient( + api_key=mock_api_key, + mock=True, + mock_responses=custom_responses + ) as client: + credits = await client.get_credits() + assert credits["remaining_credits"] == 42 + assert credits["total_credits_used"] == 58 + + @pytest.mark.asyncio + async def test_async_client_mock_responses_callable(self, mock_api_key): + """Test custom mock responses with callable values""" + def dynamic_credits(): + return {"remaining_credits": 123, "custom_field": "dynamic"} + + custom_responses = { + "/v1/credits": dynamic_credits + } + + async with AsyncClient( + api_key=mock_api_key, + mock=True, + mock_responses=custom_responses + ) as client: + credits = await client.get_credits() + assert credits["remaining_credits"] == 123 + assert credits["custom_field"] == "dynamic" + + @pytest.mark.asyncio + async def test_async_client_mock_handler_override(self, mock_api_key): + """Test custom mock handler""" + def custom_handler(method, url, kwargs): + return { + "custom_handler": True, + "method": method, + "url": url, + "has_kwargs": bool(kwargs) + } + + async with AsyncClient( + api_key=mock_api_key, + mock=True, + mock_handler=custom_handler + ) as client: + response = await client.get_credits() + assert response["custom_handler"] is True + assert response["method"] == "GET" + assert "credits" in response["url"] + + @pytest.mark.asyncio + async def test_async_client_mock_handler_fallback(self, mock_api_key): + """Test that mock handler exceptions fall back to defaults""" + def failing_handler(method, url, kwargs): + raise ValueError("Handler failed") + + async with AsyncClient( + api_key=mock_api_key, + mock=True, + mock_handler=failing_handler + ) as client: + # Should fall back to default mock response + response = await client.get_credits() + assert response["remaining_credits"] == 1000 + + +class TestAsyncMockModeEnvironment: + """Test async mock mode environment variable support""" + + @pytest.mark.asyncio + async def test_async_client_from_env_with_mock_flag(self, mock_api_key): + """Test AsyncClient.from_env with explicit mock=True""" + with patch.dict(os.environ, {"SGAI_API_KEY": mock_api_key}): + async with AsyncClient.from_env(mock=True) as client: + assert client.mock is True + + response = await client.get_credits() + assert response["remaining_credits"] == 1000 + + @pytest.mark.asyncio + async def test_async_client_from_env_without_api_key_mock_mode(self): + """Test AsyncClient.from_env in mock mode without SGAI_API_KEY set""" + with patch.dict(os.environ, {}, clear=True): + # Should work in mock mode even without API key + async with AsyncClient.from_env(mock=True) as client: + assert client.mock is True + assert client.api_key == "sgai-00000000-0000-0000-0000-000000000000" + + response = await client.get_credits() + assert response["remaining_credits"] == 1000 + + @pytest.mark.asyncio + async def test_async_client_from_env_sgai_mock_environment(self): + """Test SGAI_MOCK environment variable activation""" + test_cases = ["1", "true", "True", "TRUE", "yes", "YES", "on", "ON"] + + for mock_value in test_cases: + with patch.dict(os.environ, {"SGAI_MOCK": mock_value}, clear=True): + async with AsyncClient.from_env() as client: + assert client.mock is True, f"Failed for SGAI_MOCK={mock_value}" + + @pytest.mark.asyncio + async def test_async_client_from_env_sgai_mock_disabled(self): + """Test SGAI_MOCK environment variable disabled states""" + test_cases = ["0", "false", "False", "FALSE", "no", "NO", "off", "OFF", ""] + + for mock_value in test_cases: + with patch.dict(os.environ, {"SGAI_MOCK": mock_value}, clear=True): + try: + async with AsyncClient.from_env() as client: + # If no exception, mock should be False + assert client.mock is False, f"Failed for SGAI_MOCK={mock_value}" + except ValueError as e: + # Expected when no API key is set and mock is disabled + assert "SGAI_API_KEY environment variable not set" in str(e) + + @pytest.mark.asyncio + async def test_async_client_from_env_with_custom_responses(self, mock_api_key): + """Test AsyncClient.from_env with mock_responses parameter""" + custom_responses = { + "/v1/credits": {"remaining_credits": 999} + } + + with patch.dict(os.environ, {"SGAI_API_KEY": mock_api_key}): + async with AsyncClient.from_env( + mock=True, + mock_responses=custom_responses + ) as client: + response = await client.get_credits() + assert response["remaining_credits"] == 999 + + @pytest.mark.asyncio + async def test_async_client_from_env_with_custom_handler(self, mock_api_key): + """Test AsyncClient.from_env with mock_handler parameter""" + def custom_handler(method, url, kwargs): + return {"from_env": True, "method": method} + + with patch.dict(os.environ, {"SGAI_API_KEY": mock_api_key}): + async with AsyncClient.from_env( + mock=True, + mock_handler=custom_handler + ) as client: + response = await client.get_credits() + assert response["from_env"] is True + assert response["method"] == "GET" + + +class TestAsyncMockModeValidation: + """Test async mock mode validation and edge cases""" + + @pytest.mark.asyncio + async def test_async_client_mock_mode_disabled_by_default(self, mock_api_key): + """Test that mock mode is disabled by default""" + async with AsyncClient(api_key=mock_api_key) as client: + assert client.mock is False + + @pytest.mark.asyncio + async def test_async_client_mock_mode_with_schema(self, mock_api_key): + """Test async mock mode works with Pydantic schemas""" + class TestSchema(BaseModel): + title: str + price: float + + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # Should not raise validation errors + response = await client.smartscraper( + user_prompt="Extract data", + website_url="https://example.com", + output_schema=TestSchema + ) + assert "request_id" in response + + @pytest.mark.asyncio + async def test_async_client_mock_mode_preserves_logging(self, mock_api_key): + """Test that async mock mode preserves logging behavior""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # This should complete without errors and log mock activity + response = await client.markdownify("https://example.com") + assert "request_id" in response + + @pytest.mark.asyncio + async def test_async_client_mock_mode_context_manager(self, mock_api_key): + """Test async context manager works correctly with mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + assert client.mock is True + response = await client.get_credits() + assert response["remaining_credits"] == 1000 + + # Client should be properly closed after context exit + # Note: We can't easily test session closure without internal access + + @pytest.mark.asyncio + async def test_async_client_mock_mode_multiple_calls(self, mock_api_key): + """Test multiple async calls in mock mode""" + async with AsyncClient(api_key=mock_api_key, mock=True) as client: + # Multiple calls should all work and return consistent mock data + credits1 = await client.get_credits() + credits2 = await client.get_credits() + + assert credits1["remaining_credits"] == 1000 + assert credits2["remaining_credits"] == 1000 + + # POST calls should return different UUIDs + response1 = await client.smartscraper( + user_prompt="Test 1", + website_url="https://example.com" + ) + response2 = await client.smartscraper( + user_prompt="Test 2", + website_url="https://example.com" + ) + + assert response1["request_id"] != response2["request_id"] + assert response1["request_id"].startswith("mock-req-") + assert response2["request_id"].startswith("mock-req-") diff --git a/scrapegraph-py/tests/test_mock_client.py b/scrapegraph-py/tests/test_mock_client.py new file mode 100644 index 0000000..e3d1d62 --- /dev/null +++ b/scrapegraph-py/tests/test_mock_client.py @@ -0,0 +1,276 @@ +""" +Tests for Client mock mode functionality +""" +import os +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from pydantic import BaseModel + +from scrapegraph_py.client import Client +from tests.utils import generate_mock_api_key + + +@pytest.fixture +def mock_api_key(): + return generate_mock_api_key() + + +@pytest.fixture +def mock_uuid(): + return str(uuid4()) + + +class TestMockMode: + """Test basic mock mode functionality""" + + def test_client_mock_mode_basic(self, mock_api_key): + """Test that mock mode bypasses HTTP calls and returns stub data""" + client = Client(api_key=mock_api_key, mock=True) + + # Test credits endpoint + credits = client.get_credits() + assert credits["remaining_credits"] == 1000 + assert credits["total_credits_used"] == 0 + + # Test smartscraper POST endpoint + response = client.smartscraper( + user_prompt="Extract title", + website_url="https://example.com" + ) + assert "request_id" in response + assert response["request_id"].startswith("mock-req-") + + # Test feedback endpoint + feedback = client.submit_feedback("test-id", 5, "Great!") + assert feedback["status"] == "success" + + def test_client_mock_mode_get_endpoints(self, mock_api_key, mock_uuid): + """Test GET endpoints in mock mode""" + client = Client(api_key=mock_api_key, mock=True) + + # Test markdownify GET + md_result = client.get_markdownify(mock_uuid) + assert md_result["status"] == "completed" + assert "Mock markdown" in md_result["content"] + + # Test smartscraper GET + ss_result = client.get_smartscraper(mock_uuid) + assert ss_result["status"] == "completed" + assert "result" in ss_result + + # Test searchscraper GET + search_result = client.get_searchscraper(mock_uuid) + assert search_result["status"] == "completed" + assert "results" in search_result + + def test_client_mock_mode_crawl_endpoints(self, mock_api_key, mock_uuid): + """Test crawl-specific endpoints in mock mode""" + client = Client(api_key=mock_api_key, mock=True) + + # Test crawl POST + crawl_response = client.crawl(url="https://example.com") + assert "crawl_id" in crawl_response + assert crawl_response["crawl_id"].startswith("mock-crawl-") + + # Test crawl GET + crawl_result = client.get_crawl(mock_uuid) + assert crawl_result["status"] == "completed" + assert "pages" in crawl_result + + def test_client_mock_mode_agentic_scraper(self, mock_api_key, mock_uuid): + """Test agentic scraper endpoints in mock mode""" + client = Client(api_key=mock_api_key, mock=True) + + # Test agentic scraper POST + response = client.agenticscraper( + url="https://example.com", + steps=["click button", "extract data"] + ) + assert "request_id" in response + assert response["request_id"].startswith("mock-req-") + + # Test agentic scraper GET + result = client.get_agenticscraper(mock_uuid) + assert result["status"] == "completed" + assert "actions" in result + + +class TestMockModeCustomization: + """Test mock mode customization features""" + + def test_client_mock_responses_override(self, mock_api_key): + """Test custom mock responses via mock_responses parameter""" + custom_responses = { + "/v1/credits": {"remaining_credits": 42, "total_credits_used": 58} + } + + client = Client( + api_key=mock_api_key, + mock=True, + mock_responses=custom_responses + ) + + credits = client.get_credits() + assert credits["remaining_credits"] == 42 + assert credits["total_credits_used"] == 58 + + def test_client_mock_responses_callable(self, mock_api_key): + """Test custom mock responses with callable values""" + def dynamic_credits(): + return {"remaining_credits": 123, "custom_field": "dynamic"} + + custom_responses = { + "/v1/credits": dynamic_credits + } + + client = Client( + api_key=mock_api_key, + mock=True, + mock_responses=custom_responses + ) + + credits = client.get_credits() + assert credits["remaining_credits"] == 123 + assert credits["custom_field"] == "dynamic" + + def test_client_mock_handler_override(self, mock_api_key): + """Test custom mock handler""" + def custom_handler(method, url, kwargs): + return { + "custom_handler": True, + "method": method, + "url": url, + "has_kwargs": bool(kwargs) + } + + client = Client( + api_key=mock_api_key, + mock=True, + mock_handler=custom_handler + ) + + response = client.get_credits() + assert response["custom_handler"] is True + assert response["method"] == "GET" + assert "credits" in response["url"] + + def test_client_mock_handler_fallback(self, mock_api_key): + """Test that mock handler exceptions fall back to defaults""" + def failing_handler(method, url, kwargs): + raise ValueError("Handler failed") + + client = Client( + api_key=mock_api_key, + mock=True, + mock_handler=failing_handler + ) + + # Should fall back to default mock response + response = client.get_credits() + assert response["remaining_credits"] == 1000 + + +class TestMockModeEnvironment: + """Test mock mode environment variable support""" + + def test_client_from_env_with_mock_flag(self, mock_api_key): + """Test Client.from_env with explicit mock=True""" + with patch.dict(os.environ, {"SGAI_API_KEY": mock_api_key}): + client = Client.from_env(mock=True) + assert client.mock is True + + response = client.get_credits() + assert response["remaining_credits"] == 1000 + + def test_client_from_env_without_api_key_mock_mode(self): + """Test Client.from_env in mock mode without SGAI_API_KEY set""" + with patch.dict(os.environ, {}, clear=True): + # Should work in mock mode even without API key + client = Client.from_env(mock=True) + assert client.mock is True + assert client.api_key == "sgai-00000000-0000-0000-0000-000000000000" + + response = client.get_credits() + assert response["remaining_credits"] == 1000 + + def test_client_from_env_sgai_mock_environment(self): + """Test SGAI_MOCK environment variable activation""" + test_cases = ["1", "true", "True", "TRUE", "yes", "YES", "on", "ON"] + + for mock_value in test_cases: + with patch.dict(os.environ, {"SGAI_MOCK": mock_value}, clear=True): + client = Client.from_env() + assert client.mock is True, f"Failed for SGAI_MOCK={mock_value}" + + def test_client_from_env_sgai_mock_disabled(self): + """Test SGAI_MOCK environment variable disabled states""" + test_cases = ["0", "false", "False", "FALSE", "no", "NO", "off", "OFF", ""] + + for mock_value in test_cases: + with patch.dict(os.environ, {"SGAI_MOCK": mock_value}, clear=True): + try: + client = Client.from_env() + # If no exception, mock should be False + assert client.mock is False, f"Failed for SGAI_MOCK={mock_value}" + except ValueError as e: + # Expected when no API key is set and mock is disabled + assert "SGAI_API_KEY environment variable not set" in str(e) + + def test_client_from_env_with_custom_responses(self, mock_api_key): + """Test Client.from_env with mock_responses parameter""" + custom_responses = { + "/v1/credits": {"remaining_credits": 999} + } + + with patch.dict(os.environ, {"SGAI_API_KEY": mock_api_key}): + client = Client.from_env(mock=True, mock_responses=custom_responses) + + response = client.get_credits() + assert response["remaining_credits"] == 999 + + def test_client_from_env_with_custom_handler(self, mock_api_key): + """Test Client.from_env with mock_handler parameter""" + def custom_handler(method, url, kwargs): + return {"from_env": True, "method": method} + + with patch.dict(os.environ, {"SGAI_API_KEY": mock_api_key}): + client = Client.from_env(mock=True, mock_handler=custom_handler) + + response = client.get_credits() + assert response["from_env"] is True + assert response["method"] == "GET" + + +class TestMockModeValidation: + """Test mock mode validation and edge cases""" + + def test_client_mock_mode_disabled_by_default(self, mock_api_key): + """Test that mock mode is disabled by default""" + client = Client(api_key=mock_api_key) + assert client.mock is False + + def test_client_mock_mode_with_schema(self, mock_api_key): + """Test mock mode works with Pydantic schemas""" + class TestSchema(BaseModel): + title: str + price: float + + client = Client(api_key=mock_api_key, mock=True) + + # Should not raise validation errors + response = client.smartscraper( + user_prompt="Extract data", + website_url="https://example.com", + output_schema=TestSchema + ) + assert "request_id" in response + + def test_client_mock_mode_preserves_logging(self, mock_api_key): + """Test that mock mode preserves logging behavior""" + client = Client(api_key=mock_api_key, mock=True) + + # This should complete without errors and log mock activity + response = client.markdownify("https://example.com") + assert "request_id" in response