|
8 | 8 | MAX_BATCH_RETRIES as MAX_RETRIES, |
9 | 9 | INITIAL_RETRY_DELAY_MS as INITIAL_DELAY_MS, |
10 | 10 | } from "../constants" |
11 | | -import { getModelQueryPrefix } from "../../../shared/embeddingModels" |
12 | 11 | import { t } from "../../../i18n" |
13 | 12 |
|
14 | 13 | /** |
@@ -37,32 +36,9 @@ export class OpenAiEmbedder extends OpenAiNativeHandler implements IEmbedder { |
37 | 36 | */ |
38 | 37 | async createEmbeddings(texts: string[], model?: string): Promise<EmbeddingResponse> { |
39 | 38 | const modelToUse = model || this.defaultModelId |
40 | | - |
41 | | - // Apply model-specific query prefix if required |
42 | | - const queryPrefix = getModelQueryPrefix("openai", modelToUse) |
43 | | - const processedTexts = queryPrefix |
44 | | - ? texts.map((text, index) => { |
45 | | - const prefixedText = `${queryPrefix}${text}` |
46 | | - const estimatedTokens = Math.ceil(prefixedText.length / 4) |
47 | | - if (estimatedTokens > MAX_ITEM_TOKENS) { |
48 | | - console.warn( |
49 | | - t("embeddings:textWithPrefixExceedsTokenLimit", { |
50 | | - index, |
51 | | - estimatedTokens, |
52 | | - maxTokens: MAX_ITEM_TOKENS, |
53 | | - prefixLength: queryPrefix.length, |
54 | | - }), |
55 | | - ) |
56 | | - // Return original text without prefix to avoid exceeding limit |
57 | | - return text |
58 | | - } |
59 | | - return prefixedText |
60 | | - }) |
61 | | - : texts |
62 | | - |
63 | 39 | const allEmbeddings: number[][] = [] |
64 | 40 | const usage = { promptTokens: 0, totalTokens: 0 } |
65 | | - const remainingTexts = [...processedTexts] |
| 41 | + const remainingTexts = [...texts] |
66 | 42 |
|
67 | 43 | while (remainingTexts.length > 0) { |
68 | 44 | const currentBatch: string[] = [] |
|
0 commit comments