forked from BlockRunAI/ClawRouter
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathproxy.ts
More file actions
5582 lines (5122 loc) · 220 KB
/
Copy pathproxy.ts
File metadata and controls
5582 lines (5122 loc) · 220 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Local x402 Proxy Server
*
* Sits between OpenClaw's pi-ai (which makes standard OpenAI-format requests)
* and BlockRun's API (which requires x402 micropayments).
*
* Flow:
* pi-ai → http://localhost:{port}/v1/chat/completions
* → proxy forwards to https://blockrun.ai/api/v1/chat/completions
* → gets 402 → @x402/fetch signs payment → retries
* → streams response back to pi-ai
*
* Optimizations (v0.3.0):
* - SSE heartbeat: for streaming requests, sends headers + heartbeat immediately
* before the x402 flow, preventing OpenClaw's 10-15s timeout from firing.
* - Response dedup: hashes request bodies and caches responses for 30s,
* preventing double-charging when OpenClaw retries after timeout.
* - Smart routing: when model is "blockrun/auto", classify query and pick cheapest model.
* - Usage logging: log every request as JSON line to ~/.openclaw/blockrun/logs/
*/
import { AsyncLocalStorage } from "node:async_hooks";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
// Per-request payment tracking via AsyncLocalStorage (safe for concurrent requests).
// The x402 onAfterPaymentCreation hook writes the actual payment amount into the
// request-scoped store, and the logging code reads it after payFetch completes.
const paymentStore = new AsyncLocalStorage<{ amountUsd: number }>();
import { finished } from "node:stream";
import type { AddressInfo } from "node:net";
import { homedir } from "node:os";
import { join } from "node:path";
import { mkdir, writeFile, readFile, stat as fsStat } from "node:fs/promises";
import { readFileSync, existsSync } from "node:fs";
import { createPublicClient, http } from "viem";
import { base } from "viem/chains";
import { privateKeyToAccount } from "viem/accounts";
import { x402Client } from "@x402/fetch";
import { createPayFetchWithPreAuth } from "./payment-preauth.js";
import { registerExactEvmScheme } from "@x402/evm/exact/client";
import { toClientEvmSigner } from "@x402/evm";
import {
route,
getFallbackChain,
getFallbackChainFiltered,
filterByToolCalling,
filterByVision,
filterByExcludeList,
calculateModelCost,
DEFAULT_ROUTING_CONFIG,
type RouterOptions,
type RoutingDecision,
type RoutingConfig,
type ModelPricing,
type Tier,
} from "./router/index.js";
import { classifyByRules } from "./router/rules.js";
import {
BLOCKRUN_MODELS,
OPENCLAW_MODELS,
resolveModelAlias,
getModelContextWindow,
isReasoningModel,
supportsToolCalling,
supportsVision,
getActivePromoPrice,
} from "./models.js";
import { logUsage, type UsageEntry } from "./logger.js";
import { getStats, clearStats } from "./stats.js";
import { RequestDeduplicator } from "./dedup.js";
import { ResponseCache, type ResponseCacheConfig } from "./response-cache.js";
import { BalanceMonitor } from "./balance.js";
import type { SolanaBalanceMonitor } from "./solana-balance.js";
/** Union type for chain-agnostic balance monitoring */
type AnyBalanceMonitor = BalanceMonitor | SolanaBalanceMonitor;
import { resolvePaymentChain } from "./auth.js";
import { compressContext, shouldCompress, type NormalizedMessage } from "./compression/index.js";
// Error classes available for programmatic use but not used in proxy
// (universal free fallback means we don't throw balance errors anymore)
// import { InsufficientFundsError, EmptyWalletError } from "./errors.js";
import { USER_AGENT, VERSION } from "./version.js";
import {
SessionStore,
getSessionId,
deriveSessionId,
hashRequestContent,
type SessionConfig,
} from "./session.js";
import { checkForUpdates } from "./updater.js";
import { loadExcludeList } from "./exclude-models.js";
import { PROXY_PORT } from "./config.js";
import { SessionJournal } from "./journal.js";
import { applyUpstreamProxy } from "./upstream-proxy.js";
const BLOCKRUN_API = "https://blockrun.ai/api";
const BLOCKRUN_SOLANA_API = "https://sol.blockrun.ai/api";
const IMAGE_DIR = join(homedir(), ".openclaw", "blockrun", "images");
const AUDIO_DIR = join(homedir(), ".openclaw", "blockrun", "audio");
const VIDEO_DIR = join(homedir(), ".openclaw", "blockrun", "videos");
// Routing profile models - virtual models that trigger intelligent routing
const AUTO_MODEL = "blockrun/auto";
const ROUTING_PROFILES = new Set([
"blockrun/eco",
"eco",
"blockrun/auto",
"auto",
"blockrun/premium",
"premium",
]);
const FREE_MODELS = new Set([
"free/gpt-oss-120b",
"free/gpt-oss-20b",
"free/deepseek-v3.2",
"free/qwen3-coder-480b",
"free/glm-4.7",
"free/llama-4-maverick",
"free/qwen3-next-80b-a3b-thinking",
"free/mistral-small-4-119b",
]);
/** Pick the best available free model that isn't excluded. */
function pickFreeModel(excludeList?: Set<string>): string | undefined {
for (const m of FREE_MODELS) {
if (!excludeList?.has(m)) return m;
}
return undefined; // all free models excluded
}
// Keep backward-compat constant for places that don't have excludeList in scope
const FREE_MODEL = "free/gpt-oss-120b";
/**
* Map free/xxx model IDs to nvidia/xxx for upstream BlockRun API.
* The "free/" prefix is a ClawRouter convention for the /model picker;
* BlockRun server expects "nvidia/" prefix.
*/
function toUpstreamModelId(modelId: string): string {
if (modelId.startsWith("free/")) {
return "nvidia/" + modelId.slice("free/".length);
}
return modelId;
}
const MAX_MESSAGES = 200; // BlockRun API limit - truncate older messages if exceeded
const CONTEXT_LIMIT_KB = 5120; // Server-side limit: 5MB in KB
const HEARTBEAT_INTERVAL_MS = 2_000;
const DEFAULT_REQUEST_TIMEOUT_MS = 180_000; // 3 minutes (allows for on-chain tx + LLM response)
const PER_MODEL_TIMEOUT_MS = 60_000; // 60s per individual model attempt (fallback to next on exceed)
const MAX_FALLBACK_ATTEMPTS = 5; // Maximum models to try in fallback chain (increased from 3 to ensure cheap models are tried)
const HEALTH_CHECK_TIMEOUT_MS = 2_000; // Timeout for checking existing proxy
const RATE_LIMIT_COOLDOWN_MS = 60_000; // 60 seconds cooldown for rate-limited models
const OVERLOAD_COOLDOWN_MS = 15_000; // 15 seconds cooldown for overloaded providers
const PORT_RETRY_ATTEMPTS = 5; // Max attempts to bind port (handles TIME_WAIT)
const PORT_RETRY_DELAY_MS = 1_000; // Delay between retry attempts
const MODEL_BODY_READ_TIMEOUT_MS = 300_000; // 5 minutes for model responses (reasoning models are slow)
const ERROR_BODY_READ_TIMEOUT_MS = 30_000; // 30 seconds for error/partner body reads
async function readBodyWithTimeout(
body: ReadableStream<Uint8Array> | null,
timeoutMs: number = MODEL_BODY_READ_TIMEOUT_MS,
): Promise<Uint8Array[]> {
if (!body) return [];
const reader = body.getReader();
const chunks: Uint8Array[] = [];
let timer: ReturnType<typeof setTimeout> | undefined;
try {
while (true) {
const result = await Promise.race([
reader.read(),
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error("Body read timeout")), timeoutMs);
}),
]);
clearTimeout(timer);
if (result.done) break;
chunks.push(result.value);
}
} finally {
clearTimeout(timer);
reader.releaseLock();
}
return chunks;
}
/**
* Transform upstream payment errors into user-friendly messages.
* Parses the raw x402 error and formats it nicely.
*/
export function transformPaymentError(errorBody: string): string {
try {
// Try to parse the error JSON
const parsed = JSON.parse(errorBody) as {
error?: string;
details?: string;
// blockrun-sol (Solana) format uses code+debug instead of details
code?: string;
debug?: string;
payer?: string;
};
// Check if this is a payment verification error
if (parsed.error === "Payment verification failed" && parsed.details) {
// Extract the nested JSON from details
// Format: "Verification failed: {json}\n"
const match = parsed.details.match(/Verification failed:\s*(\{.*\})/s);
if (match) {
const innerJson = JSON.parse(match[1]) as {
invalidMessage?: string;
invalidReason?: string;
payer?: string;
};
if (innerJson.invalidReason === "insufficient_funds" && innerJson.invalidMessage) {
// Parse "insufficient balance: 251 < 11463"
const balanceMatch = innerJson.invalidMessage.match(
/insufficient balance:\s*(\d+)\s*<\s*(\d+)/i,
);
if (balanceMatch) {
const currentMicros = parseInt(balanceMatch[1], 10);
const requiredMicros = parseInt(balanceMatch[2], 10);
const currentUSD = (currentMicros / 1_000_000).toFixed(6);
const requiredUSD = (requiredMicros / 1_000_000).toFixed(6);
const wallet = innerJson.payer || "unknown";
const shortWallet =
wallet.length > 12 ? `${wallet.slice(0, 6)}...${wallet.slice(-4)}` : wallet;
return JSON.stringify({
error: {
message: `Insufficient USDC balance. Current: $${currentUSD}, Required: ~$${requiredUSD}`,
type: "insufficient_funds",
wallet: wallet,
current_balance_usd: currentUSD,
required_usd: requiredUSD,
help: `Fund wallet ${shortWallet} with USDC on Base, or use free model: /model free`,
},
});
}
}
// Handle invalid_payload errors (signature issues, malformed payment)
if (innerJson.invalidReason === "invalid_payload") {
return JSON.stringify({
error: {
message: "Payment signature invalid. This may be a temporary issue.",
type: "invalid_payload",
help: "Try again. If this persists, reinstall ClawRouter: curl -fsSL https://blockrun.ai/ClawRouter-update | bash",
},
});
}
// Handle transaction simulation failures (Solana on-chain validation)
if (innerJson.invalidReason === "transaction_simulation_failed") {
console.error(
`[ClawRouter] Solana transaction simulation failed: ${innerJson.invalidMessage || "unknown"}`,
);
return JSON.stringify({
error: {
message: "Solana payment simulation failed. Retrying with a different model.",
type: "transaction_simulation_failed",
help: "This is usually temporary. If it persists, check your Solana USDC balance or try: /model free",
},
});
}
}
}
// Handle code=PAYMENT_INVALID + debug format (used by blockrun-sol, can also
// appear from blockrun Base when CDP returns non-200 with structured JSON body)
if (
parsed.error === "Payment verification failed" &&
parsed.code === "PAYMENT_INVALID" &&
parsed.debug
) {
const debugLower = parsed.debug.toLowerCase();
const wallet = parsed.payer || "unknown";
const shortWallet =
wallet.length > 12 ? `${wallet.slice(0, 6)}...${wallet.slice(-4)}` : wallet;
// Detect chain from payer address format (0x = EVM, else Solana)
const chain = wallet.startsWith("0x") ? "Base" : "Solana";
if (debugLower.includes("insufficient")) {
return JSON.stringify({
error: {
message: `Insufficient ${chain} USDC balance.`,
type: "insufficient_funds",
wallet,
help:
chain === "Solana"
? `Fund wallet ${shortWallet} with USDC on Solana, or switch to Base: /wallet base`
: `Fund wallet ${shortWallet} with USDC on Base, or use free model: /model free`,
},
});
}
if (
debugLower.includes("transaction_simulation_failed") ||
debugLower.includes("simulation")
) {
console.error(`[ClawRouter] ${chain} transaction simulation failed: ${parsed.debug}`);
return JSON.stringify({
error: {
message: `${chain} payment simulation failed. Retrying with a different model.`,
type: "transaction_simulation_failed",
help: "This is usually temporary. If it persists, try: /model free",
},
});
}
if (debugLower.includes("invalid signature") || debugLower.includes("invalid_signature")) {
return JSON.stringify({
error: {
message: `${chain} payment signature invalid.`,
type: "invalid_payload",
help: "Try again. If this persists, reinstall ClawRouter: curl -fsSL https://blockrun.ai/ClawRouter-update | bash",
},
});
}
if (debugLower.includes("expired")) {
return JSON.stringify({
error: {
message: `${chain} payment expired. Retrying.`,
type: "expired",
help: "This is usually temporary.",
},
});
}
// Unknown verification error — surface the debug reason
console.error(
`[ClawRouter] ${chain} payment verification failed: ${parsed.debug} payer=${wallet}`,
);
return JSON.stringify({
error: {
message: `${chain} payment verification failed: ${parsed.debug}`,
type: "payment_invalid",
wallet,
help:
chain === "Solana"
? "Try again or switch to Base: /wallet base"
: "Try again. If this persists, try: /model free",
},
});
}
// Handle settlement failures (gas estimation, on-chain errors)
if (
parsed.error === "Settlement failed" ||
parsed.error === "Payment settlement failed" ||
parsed.details?.includes("Settlement failed") ||
parsed.details?.includes("transaction_simulation_failed")
) {
const details = parsed.details || "";
const gasError = details.includes("unable to estimate gas");
return JSON.stringify({
error: {
message: gasError
? "Payment failed: network congestion or gas issue. Try again."
: "Payment settlement failed. Try again in a moment.",
type: "settlement_failed",
help: "This is usually temporary. If it persists, try: /model free",
},
});
}
} catch {
// If parsing fails, return original
}
return errorBody;
}
/**
* Semantic error categories from upstream provider responses.
* Used to distinguish auth failures from rate limits from server errors
* so each category can be handled independently without cross-contamination.
*/
export type ErrorCategory =
| "auth_failure" // 401, 403: Wrong key or forbidden — don't retry with same key
| "quota_exceeded" // 403 with plan/quota body: Plan limit hit
| "rate_limited" // 429: Actual throttling — 60s cooldown
| "overloaded" // 529, 503+overload body: Provider capacity — 15s cooldown
| "server_error" // 5xx general: Transient — fallback immediately
| "payment_error" // 402: x402 payment or funds issue
| "config_error"; // 400, 413: Bad request content — skip this model
/**
* Classify an upstream error response into a semantic category.
* Returns null if the status+body is not a provider-side issue worth retrying.
*/
export function categorizeError(status: number, body: string): ErrorCategory | null {
if (status === 401) return "auth_failure";
if (status === 402) return "payment_error";
if (status === 403) {
if (/plan.*limit|quota.*exceeded|subscription|allowance/i.test(body)) return "quota_exceeded";
return "auth_failure"; // generic 403 = forbidden = likely auth issue
}
if (status === 429) return "rate_limited";
if (status === 529) return "overloaded";
if (status === 503 && /overload|capacity|too.*many.*request/i.test(body)) return "overloaded";
if (status >= 500) return "server_error";
if (status === 400 || status === 413) {
// Only fallback on content-size or billing patterns; bare 400 = our bug, don't cycle
if (PROVIDER_ERROR_PATTERNS.some((p) => p.test(body))) return "config_error";
return null;
}
return null;
}
/**
* Track rate-limited models to avoid hitting them again.
* Maps model ID to the timestamp when the rate limit was hit.
*/
const rateLimitedModels = new Map<string, number>();
/** Per-model overload tracking (529/503 capacity errors) — shorter cooldown than rate limits. */
const overloadedModels = new Map<string, number>();
/** Per-model error category counts (in-memory, resets on restart). */
type ProviderErrorCounts = {
auth_failure: number;
quota_exceeded: number;
rate_limited: number;
overloaded: number;
server_error: number;
payment_error: number;
config_error: number;
};
const perProviderErrors = new Map<string, ProviderErrorCounts>();
/** Record an error category hit for a model. */
function recordProviderError(modelId: string, category: ErrorCategory): void {
if (!perProviderErrors.has(modelId)) {
perProviderErrors.set(modelId, {
auth_failure: 0,
quota_exceeded: 0,
rate_limited: 0,
overloaded: 0,
server_error: 0,
payment_error: 0,
config_error: 0,
});
}
perProviderErrors.get(modelId)![category]++;
}
/**
* Check if a model is currently rate-limited (in cooldown period).
*/
function isRateLimited(modelId: string): boolean {
const hitTime = rateLimitedModels.get(modelId);
if (!hitTime) return false;
const elapsed = Date.now() - hitTime;
if (elapsed >= RATE_LIMIT_COOLDOWN_MS) {
rateLimitedModels.delete(modelId);
return false;
}
return true;
}
/**
* Mark a model as rate-limited.
*/
function markRateLimited(modelId: string): void {
rateLimitedModels.set(modelId, Date.now());
console.log(`[ClawRouter] Model ${modelId} rate-limited, will deprioritize for 60s`);
}
/**
* Mark a model as temporarily overloaded (529/503 capacity).
* Shorter cooldown than rate limits since capacity restores quickly.
*/
function markOverloaded(modelId: string): void {
overloadedModels.set(modelId, Date.now());
console.log(`[ClawRouter] Model ${modelId} overloaded, will deprioritize for 15s`);
}
/** Check if a model is in its overload cooldown period. */
function isOverloaded(modelId: string): boolean {
const hitTime = overloadedModels.get(modelId);
if (!hitTime) return false;
if (Date.now() - hitTime >= OVERLOAD_COOLDOWN_MS) {
overloadedModels.delete(modelId);
return false;
}
return true;
}
/**
* Reorder models to put rate-limited ones at the end.
*/
function prioritizeNonRateLimited(models: string[]): string[] {
const available: string[] = [];
const degraded: string[] = [];
for (const model of models) {
if (isRateLimited(model) || isOverloaded(model)) {
degraded.push(model);
} else {
available.push(model);
}
}
return [...available, ...degraded];
}
/**
* Check if response socket is writable (prevents write-after-close errors).
* Returns true only if all conditions are safe for writing.
*/
function canWrite(res: ServerResponse): boolean {
return (
!res.writableEnded &&
!res.destroyed &&
res.socket !== null &&
!res.socket.destroyed &&
res.socket.writable
);
}
/**
* Safe write with backpressure handling.
* Returns true if write succeeded, false if socket is closed or write failed.
*/
function safeWrite(res: ServerResponse, data: string | Buffer): boolean {
if (!canWrite(res)) {
const bytes = typeof data === "string" ? Buffer.byteLength(data) : data.length;
console.warn(`[ClawRouter] safeWrite: socket not writable, dropping ${bytes} bytes`);
return false;
}
return res.write(data);
}
// Extra buffer for balance check (on top of estimateAmount's 20% buffer)
// Total effective buffer: 1.2 * 1.5 = 1.8x (80% safety margin)
// This prevents x402 payment failures after streaming headers are sent,
// which would trigger OpenClaw's 5-24 hour billing cooldown.
const BALANCE_CHECK_BUFFER = 1.5;
/**
* Get the proxy port from pre-loaded configuration.
* Port is validated at module load time, this just returns the cached value.
*/
export function getProxyPort(): number {
return PROXY_PORT;
}
/**
* Check if a proxy is already running on the given port.
* Returns the wallet address if running, undefined otherwise.
*/
async function checkExistingProxy(
port: number,
): Promise<{ wallet: string; paymentChain?: string } | undefined> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), HEALTH_CHECK_TIMEOUT_MS);
try {
const response = await fetch(`http://127.0.0.1:${port}/health`, {
signal: controller.signal,
});
clearTimeout(timeoutId);
if (response.ok) {
const data = (await response.json()) as {
status?: string;
wallet?: string;
paymentChain?: string;
};
if (data.status === "ok" && data.wallet) {
return { wallet: data.wallet, paymentChain: data.paymentChain };
}
}
return undefined;
} catch {
clearTimeout(timeoutId);
return undefined;
}
}
/**
* Error patterns that indicate a provider-side issue (not user's fault).
* These errors should trigger fallback to the next model in the chain.
*/
const PROVIDER_ERROR_PATTERNS = [
/billing/i,
/insufficient.*balance/i,
/credits/i,
/quota.*exceeded/i,
/rate.*limit/i,
/model.*unavailable/i,
/model.*not.*available/i,
/service.*unavailable/i,
/capacity/i,
/overloaded/i,
/temporarily.*unavailable/i,
/api.*key.*invalid/i,
/authentication.*failed/i,
/request too large/i,
/request.*size.*exceeds/i,
/payload too large/i,
/payment.*verification.*failed/i,
/model.*not.*allowed/i,
/unknown.*model/i,
/reasoning_content.*missing/i, // Thinking model multi-turn: missing reasoning_content → fallback
/thinking.*reasoning_content/i,
];
/**
* "Successful" response bodies that are actually provider degradation placeholders.
* Some upstream providers occasionally return these with HTTP 200.
*/
const DEGRADED_RESPONSE_PATTERNS = [
/the ai service is temporarily overloaded/i,
/service is temporarily overloaded/i,
/please try again in a moment/i,
];
/**
* Known low-quality loop signatures seen during provider degradation windows.
*/
const DEGRADED_LOOP_PATTERNS = [
/the boxed is the response\./i,
/the response is the text\./i,
/the final answer is the boxed\./i,
];
function extractAssistantContent(payload: unknown): string | undefined {
if (!payload || typeof payload !== "object") return undefined;
const record = payload as Record<string, unknown>;
const choices = record.choices;
if (!Array.isArray(choices) || choices.length === 0) return undefined;
const firstChoice = choices[0];
if (!firstChoice || typeof firstChoice !== "object") return undefined;
const choice = firstChoice as Record<string, unknown>;
const message = choice.message;
if (!message || typeof message !== "object") return undefined;
const content = (message as Record<string, unknown>).content;
return typeof content === "string" ? content : undefined;
}
function hasKnownLoopSignature(text: string): boolean {
const matchCount = DEGRADED_LOOP_PATTERNS.reduce(
(count, pattern) => (pattern.test(text) ? count + 1 : count),
0,
);
if (matchCount >= 2) return true;
// Generic repetitive loop fallback for short repeated lines.
const lines = text
.split(/\r?\n/)
.map((line) => line.trim())
.filter(Boolean);
if (lines.length < 8) return false;
const counts = new Map<string, number>();
for (const line of lines) {
counts.set(line, (counts.get(line) ?? 0) + 1);
}
const maxRepeat = Math.max(...counts.values());
const uniqueRatio = counts.size / lines.length;
return maxRepeat >= 3 && uniqueRatio <= 0.45;
}
/**
* Detect degraded 200-response payloads that should trigger model fallback.
* Returns a short reason when fallback should happen, otherwise undefined.
*/
export function detectDegradedSuccessResponse(body: string): string | undefined {
const trimmed = body.trim();
if (!trimmed) return undefined;
// Plain-text placeholder response.
if (DEGRADED_RESPONSE_PATTERNS.some((pattern) => pattern.test(trimmed))) {
return "degraded response: overloaded placeholder";
}
// Plain-text looping garbage response.
if (hasKnownLoopSignature(trimmed)) {
return "degraded response: repetitive loop output";
}
try {
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
// Some providers return JSON error payloads with HTTP 200.
const errorField = parsed.error;
let errorText = "";
if (typeof errorField === "string") {
errorText = errorField;
} else if (errorField && typeof errorField === "object") {
const errObj = errorField as Record<string, unknown>;
errorText = [
typeof errObj.message === "string" ? errObj.message : "",
typeof errObj.type === "string" ? errObj.type : "",
typeof errObj.code === "string" ? errObj.code : "",
]
.filter(Boolean)
.join(" ");
}
if (errorText && PROVIDER_ERROR_PATTERNS.some((pattern) => pattern.test(errorText))) {
return `degraded response: ${errorText.slice(0, 120)}`;
}
// Detect empty-turn responses: model returned 200 but no content and no tool calls.
// Happens when models like gemini-3.1-flash-lite receive complex agentic requests
// (e.g. Roo Code tool schemas) and produce zero output instead of refusing.
const choices = parsed.choices;
if (Array.isArray(choices) && choices.length > 0) {
const choice = choices[0] as Record<string, unknown>;
const msg = (choice.message ?? choice.delta) as Record<string, unknown> | undefined;
if (msg) {
const content = msg.content;
const toolCalls = msg.tool_calls;
const hasContent = typeof content === "string" && content.trim().length > 0;
const hasToolCalls = Array.isArray(toolCalls) && toolCalls.length > 0;
const finishReason = choice.finish_reason as string | null | undefined;
if (!hasContent && !hasToolCalls && finishReason === "stop") {
return "degraded response: empty turn (no content or tool calls)";
}
}
}
// Successful wrapper with bad assistant content.
const assistantContent = extractAssistantContent(parsed);
if (!assistantContent) return undefined;
if (DEGRADED_RESPONSE_PATTERNS.some((pattern) => pattern.test(assistantContent))) {
return "degraded response: overloaded assistant content";
}
if (hasKnownLoopSignature(assistantContent)) {
return "degraded response: repetitive assistant loop";
}
} catch {
// Not JSON - handled by plaintext checks above.
}
return undefined;
}
/**
* Valid message roles for OpenAI-compatible APIs.
* Some clients send non-standard roles (e.g., "developer" instead of "system").
*/
const VALID_ROLES = new Set(["system", "user", "assistant", "tool", "function"]);
/**
* Role mappings for non-standard roles.
* Maps client-specific roles to standard OpenAI roles.
*/
const ROLE_MAPPINGS: Record<string, string> = {
developer: "system", // OpenAI's newer API uses "developer" for system messages
model: "assistant", // Some APIs use "model" instead of "assistant"
};
type ChatMessage = { role: string; content: string | unknown };
/**
* Anthropic tool ID pattern: only alphanumeric, underscore, and hyphen allowed.
* Error: "messages.X.content.Y.tool_use.id: String should match pattern '^[a-zA-Z0-9_-]+$'"
*/
const VALID_TOOL_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
/**
* Sanitize a tool ID to match Anthropic's required pattern.
* Replaces invalid characters with underscores.
*/
function sanitizeToolId(id: string | undefined): string | undefined {
if (!id || typeof id !== "string") return id;
if (VALID_TOOL_ID_PATTERN.test(id)) return id;
// Replace invalid characters with underscores
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
}
/**
* Type for messages with tool calls (OpenAI format).
*/
type MessageWithTools = ChatMessage & {
tool_calls?: Array<{ id?: string; type?: string; function?: unknown }>;
tool_call_id?: string;
};
/**
* Type for content blocks that may contain tool IDs (Anthropic format in OpenAI wrapper).
*/
type ContentBlock = {
type?: string;
id?: string;
tool_use_id?: string;
[key: string]: unknown;
};
/**
* Sanitize all tool IDs in messages to match Anthropic's pattern.
* Handles both OpenAI format (tool_calls, tool_call_id) and content block formats.
*/
function sanitizeToolIds(messages: ChatMessage[]): ChatMessage[] {
if (!messages || messages.length === 0) return messages;
let hasChanges = false;
const sanitized = messages.map((msg) => {
const typedMsg = msg as MessageWithTools;
let msgChanged = false;
let newMsg = { ...msg } as MessageWithTools;
// Sanitize tool_calls[].id in assistant messages
if (typedMsg.tool_calls && Array.isArray(typedMsg.tool_calls)) {
const newToolCalls = typedMsg.tool_calls.map((tc) => {
if (tc.id && typeof tc.id === "string") {
const sanitized = sanitizeToolId(tc.id);
if (sanitized !== tc.id) {
msgChanged = true;
return { ...tc, id: sanitized };
}
}
return tc;
});
if (msgChanged) {
newMsg = { ...newMsg, tool_calls: newToolCalls };
}
}
// Sanitize tool_call_id in tool messages
if (typedMsg.tool_call_id && typeof typedMsg.tool_call_id === "string") {
const sanitized = sanitizeToolId(typedMsg.tool_call_id);
if (sanitized !== typedMsg.tool_call_id) {
msgChanged = true;
newMsg = { ...newMsg, tool_call_id: sanitized };
}
}
// Sanitize content blocks if content is an array (Anthropic-style content)
if (Array.isArray(typedMsg.content)) {
const newContent = (typedMsg.content as ContentBlock[]).map((block) => {
if (!block || typeof block !== "object") return block;
let blockChanged = false;
let newBlock = { ...block };
// tool_use blocks have "id"
if (block.type === "tool_use" && block.id && typeof block.id === "string") {
const sanitized = sanitizeToolId(block.id);
if (sanitized !== block.id) {
blockChanged = true;
newBlock = { ...newBlock, id: sanitized };
}
}
// tool_result blocks have "tool_use_id"
if (
block.type === "tool_result" &&
block.tool_use_id &&
typeof block.tool_use_id === "string"
) {
const sanitized = sanitizeToolId(block.tool_use_id);
if (sanitized !== block.tool_use_id) {
blockChanged = true;
newBlock = { ...newBlock, tool_use_id: sanitized };
}
}
if (blockChanged) {
msgChanged = true;
return newBlock;
}
return block;
});
if (msgChanged) {
newMsg = { ...newMsg, content: newContent };
}
}
if (msgChanged) {
hasChanges = true;
return newMsg;
}
return msg;
});
return hasChanges ? sanitized : messages;
}
/**
* Normalize message roles to standard OpenAI format.
* Converts non-standard roles (e.g., "developer") to valid ones.
*/
function normalizeMessageRoles(messages: ChatMessage[]): ChatMessage[] {
if (!messages || messages.length === 0) return messages;
let hasChanges = false;
const normalized = messages.map((msg) => {
if (VALID_ROLES.has(msg.role)) return msg;
const mappedRole = ROLE_MAPPINGS[msg.role];
if (mappedRole) {
hasChanges = true;
return { ...msg, role: mappedRole };
}
// Unknown role - default to "user" to avoid API errors
hasChanges = true;
return { ...msg, role: "user" };
});
return hasChanges ? normalized : messages;
}
/**
* Normalize messages for Google models.
* Google's Gemini API requires the first non-system message to be from "user".
* If conversation starts with "assistant"/"model", prepend a placeholder user message.
*/
function normalizeMessagesForGoogle(messages: ChatMessage[]): ChatMessage[] {
if (!messages || messages.length === 0) return messages;
// Find first non-system message
let firstNonSystemIdx = -1;
for (let i = 0; i < messages.length; i++) {
if (messages[i].role !== "system") {
firstNonSystemIdx = i;
break;
}
}
// If no non-system messages, return as-is
if (firstNonSystemIdx === -1) return messages;
const firstRole = messages[firstNonSystemIdx].role;
// If first non-system message is already "user", no change needed
if (firstRole === "user") return messages;
// If first non-system message is "assistant" or "model", prepend a user message
if (firstRole === "assistant" || firstRole === "model") {
const normalized = [...messages];
normalized.splice(firstNonSystemIdx, 0, {
role: "user",
content: "(continuing conversation)",
});
return normalized;
}
return messages;
}
/**
* Check if a model is a Google model that requires message normalization.
*/
function isGoogleModel(modelId: string): boolean {
return modelId.startsWith("google/") || modelId.startsWith("gemini");
}
/**
* Extended message type for thinking-enabled conversations.
*/
type ExtendedChatMessage = ChatMessage & {
tool_calls?: unknown[];
reasoning_content?: unknown;
};
/**
* Normalize messages for thinking-enabled requests.
* When thinking/extended_thinking is enabled, ALL assistant messages in history
* must have reasoning_content (can be empty string if not present).
*
* Reasoning models like Kimi K2.5, DeepSeek-R1, etc. strip their thinking tokens
* before we forward the response to the client. So when the client sends back the
* assistant message in a follow-up turn, it lacks reasoning_content. These models
* reject the multi-turn history with 400 if any assistant message is missing it.
*
* Previously only tool-call messages were patched — this missed plain text assistant
* messages, causing 100% failures on existing (multi-turn) chats.
*
* Error examples:
* "400 thinking is enabled but reasoning_content is missing in assistant tool call message"
* "400 thinking is enabled but reasoning_content is missing in assistant message"
*/
export function normalizeMessagesForThinking(
messages: ExtendedChatMessage[],
): ExtendedChatMessage[] {
if (!messages || messages.length === 0) return messages;
let hasChanges = false;
const normalized = messages.map((msg) => {
// Skip if not assistant or already has reasoning_content
if (msg.role !== "assistant" || msg.reasoning_content !== undefined) {
return msg;
}
// Add reasoning_content: "" to ALL assistant messages.
// Reasoning models require it on every assistant turn in history, not just tool-call turns.
hasChanges = true;
return { ...msg, reasoning_content: "" };
});