Skip to content

Commit 82d1645

Browse files
HenryHengZJHenryHengZJ
authored
Bugfix/Update file loader (#4420)
* update file loader * delete temp for o1 * fix tavily
1 parent eadf1b1 commit 82d1645

File tree

6 files changed

+119
-28
lines changed

6 files changed

+119
-28
lines changed

‎packages/components/nodes/chatmodels/AzureChatOpenAI/AzureChatOpenAI.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ class AzureChatOpenAI_ChatModels implements INode {
237237
console.error('Error parsing base options', exception)
238238
}
239239
}
240-
if (modelName === 'o3-mini') {
240+
if (modelName === 'o3-mini' || modelName.includes('o1')) {
241241
delete obj.temperature
242242
}
243243
if ((modelName.includes('o1') || modelName.includes('o3')) && reasoningEffort) {

‎packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ class ChatOpenAI_ChatModels implements INode {
243243
streaming: streaming ?? true
244244
}
245245

246-
if (modelName.includes('o3')) {
246+
if (modelName.includes('o3') || modelName.includes('o1')) {
247247
delete obj.temperature
248248
}
249249
if ((modelName.includes('o1') || modelName.includes('o3')) && reasoningEffort) {

‎packages/components/nodes/documentloaders/File/File.ts‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ class File_DocumentLoaders implements INode {
4747
},
4848
{
4949
label: 'Pdf Usage',
50-
name: 'pdfUsage',
50+
name: 'usage',
5151
type: 'options',
5252
description: 'Only when loading PDF files',
5353
options: [
@@ -64,6 +64,14 @@ class File_DocumentLoaders implements INode {
6464
optional: true,
6565
additionalParams: true
6666
},
67+
{
68+
label: 'Use Legacy Build',
69+
name: 'legacyBuild',
70+
type: 'boolean',
71+
description: 'Use legacy build for PDF compatibility issues',
72+
optional: true,
73+
additionalParams: true
74+
},
6775
{
6876
label: 'JSONL Pointer Extraction',
6977
name: 'pointerName',
@@ -113,7 +121,8 @@ class File_DocumentLoaders implements INode {
113121
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
114122
const fileBase64 = nodeData.inputs?.file as string
115123
const metadata = nodeData.inputs?.metadata
116-
const pdfUsage = nodeData.inputs?.pdfUsage
124+
const pdfUsage = nodeData.inputs?.pdfUsage || nodeData.inputs?.usage
125+
const legacyBuild = nodeData.inputs?.legacyBuild as boolean
117126
const pointerName = nodeData.inputs?.pointerName as string
118127
const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
119128
const output = nodeData.outputs?.output as string
@@ -173,10 +182,21 @@ class File_DocumentLoaders implements INode {
173182
const match = file.match(/^data:([A-Za-z-+\/]+);base64,/)
174183

175184
if (!match) {
176-
fileBlobs.push({
177-
blob,
178-
ext: extension
179-
})
185+
// Fallback: check if there's a filename pattern at the end
186+
const filenameMatch = file.match(/,filename:(.+\.\w+)$/)
187+
if (filenameMatch && filenameMatch[1]) {
188+
const filename = filenameMatch[1]
189+
const fileExt = filename.split('.').pop() || ''
190+
fileBlobs.push({
191+
blob,
192+
ext: fileExt
193+
})
194+
} else {
195+
fileBlobs.push({
196+
blob,
197+
ext: extension
198+
})
199+
}
180200
} else {
181201
const mimeType = match[1]
182202
fileBlobs.push({
@@ -199,9 +219,18 @@ class File_DocumentLoaders implements INode {
199219
pdf: (blob) =>
200220
pdfUsage === 'perFile'
201221
? // @ts-ignore
202-
new PDFLoader(blob, { splitPages: false, pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') })
222+
new PDFLoader(blob, {
223+
splitPages: false,
224+
pdfjs: () =>
225+
// @ts-ignore
226+
legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
227+
})
203228
: // @ts-ignore
204-
new PDFLoader(blob, { pdfjs: () => import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js') }),
229+
new PDFLoader(blob, {
230+
pdfjs: () =>
231+
// @ts-ignore
232+
legacyBuild ? import('pdfjs-dist/legacy/build/pdf.js') : import('pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js')
233+
}),
205234
'': (blob) => new TextLoader(blob)
206235
})
207236
let docs = []

‎packages/components/nodes/tools/TavilyAPI/TavilyAPI.ts‎

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,12 @@ class TavilyAPI_Tools implements INode {
1818
constructor() {
1919
this.label = 'Tavily API'
2020
this.name = 'tavilyAPI'
21-
this.version = 1.1
21+
this.version = 1.2
2222
this.type = 'TavilyAPI'
2323
this.icon = 'tavily.svg'
2424
this.category = 'Tools'
2525
this.description = 'Wrapper around TavilyAPI - A specialized search engine designed for LLMs and AI agents'
2626
this.inputs = [
27-
{
28-
label: 'Query',
29-
name: 'query',
30-
type: 'string',
31-
optional: false,
32-
description: 'The search query to execute with Tavily',
33-
additionalParams: true
34-
},
3527
{
3628
label: 'Topic',
3729
name: 'topic',
@@ -165,7 +157,6 @@ class TavilyAPI_Tools implements INode {
165157
const credentialData = await getCredentialData(nodeData.credential ?? '', options)
166158
const tavilyApiKey = getCredentialParam('tavilyApiKey', credentialData, nodeData)
167159

168-
const query = nodeData.inputs?.query as string
169160
const topic = nodeData.inputs?.topic as string
170161
const searchDepth = nodeData.inputs?.searchDepth as string
171162
const chunksPerSource = nodeData.inputs?.chunksPerSource as number
@@ -181,14 +172,13 @@ class TavilyAPI_Tools implements INode {
181172

182173
const config: any = {
183174
apiKey: tavilyApiKey,
184-
query,
185175
topic,
186176
searchDepth,
187177
maxResults,
188-
includeAnswer,
189-
includeRawContent,
190-
includeImages,
191-
includeImageDescriptions
178+
includeAnswer: includeAnswer || undefined,
179+
includeRawContent: includeRawContent || undefined,
180+
includeImages: includeImages || undefined,
181+
includeImageDescriptions: includeImageDescriptions || undefined
192182
}
193183

194184
if (chunksPerSource) config.chunksPerSource = chunksPerSource

‎packages/server/src/utils/createAttachment.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,28 @@ export const createFileAttachment = async (req: Request) => {
4646
throw new InternalFlowiseError(StatusCodes.NOT_FOUND, `Chatflow ${chatflowid} not found`)
4747
}
4848

49+
// Parse chatbot configuration to get file upload settings
50+
let pdfConfig = {
51+
usage: 'perPage',
52+
legacyBuild: false
53+
}
54+
55+
if (chatflow.chatbotConfig) {
56+
try {
57+
const chatbotConfig = JSON.parse(chatflow.chatbotConfig)
58+
if (chatbotConfig?.fullFileUpload?.pdfFile) {
59+
if (chatbotConfig.fullFileUpload.pdfFile.usage) {
60+
pdfConfig.usage = chatbotConfig.fullFileUpload.pdfFile.usage
61+
}
62+
if (chatbotConfig.fullFileUpload.pdfFile.legacyBuild !== undefined) {
63+
pdfConfig.legacyBuild = chatbotConfig.fullFileUpload.pdfFile.legacyBuild
64+
}
65+
}
66+
} catch (e) {
67+
// Use default PDF config if parsing fails
68+
}
69+
}
70+
4971
// Find FileLoader node
5072
const fileLoaderComponent = appServer.nodesPool.componentNodes['fileLoader']
5173
const fileLoaderNodeInstanceFilePath = fileLoaderComponent.filePath as string
@@ -93,6 +115,12 @@ export const createFileAttachment = async (req: Request) => {
93115
outputs: { output: 'document' }
94116
}
95117

118+
// Apply PDF specific configuration if this is a PDF file
119+
if (fileInputField === 'pdfFile') {
120+
nodeData.inputs.usage = pdfConfig.usage
121+
nodeData.inputs.legacyBuild = pdfConfig.legacyBuild as unknown as string
122+
}
123+
96124
let content = ''
97125

98126
if (isBase64) {

‎packages/ui/src/ui-component/extended/FileUpload.jsx‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { enqueueSnackbar as enqueueSnackbarAction, closeSnackbar as closeSnackba
55
import parser from 'html-react-parser'
66

77
// material-ui
8-
import { Button, Box, Typography } from '@mui/material'
8+
import { Button, Box, Typography, FormControl, RadioGroup, FormControlLabel, Radio } from '@mui/material'
99
import { IconX, IconBulb } from '@tabler/icons-react'
1010

1111
// Project import
@@ -31,7 +31,9 @@ const availableFileTypes = [
3131
{ name: 'PDF', ext: 'application/pdf' },
3232
{ name: 'SQL', ext: 'application/sql' },
3333
{ name: 'Text File', ext: 'text/plain' },
34-
{ name: 'XML', ext: 'application/xml' }
34+
{ name: 'XML', ext: 'application/xml' },
35+
{ name: 'DOC', ext: 'application/msword' },
36+
{ name: 'DOCX', ext: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }
3537
]
3638

3739
const FileUpload = ({ dialogProps }) => {
@@ -45,6 +47,8 @@ const FileUpload = ({ dialogProps }) => {
4547
const [fullFileUpload, setFullFileUpload] = useState(false)
4648
const [allowedFileTypes, setAllowedFileTypes] = useState([])
4749
const [chatbotConfig, setChatbotConfig] = useState({})
50+
const [pdfUsage, setPdfUsage] = useState('perPage')
51+
const [pdfLegacyBuild, setPdfLegacyBuild] = useState(false)
4852

4953
const handleChange = (value) => {
5054
setFullFileUpload(value)
@@ -59,11 +63,23 @@ const FileUpload = ({ dialogProps }) => {
5963
}
6064
}
6165

66+
const handlePdfUsageChange = (event) => {
67+
setPdfUsage(event.target.value)
68+
}
69+
70+
const handleLegacyBuildChange = (value) => {
71+
setPdfLegacyBuild(value)
72+
}
73+
6274
const onSave = async () => {
6375
try {
6476
const value = {
6577
status: fullFileUpload,
66-
allowedUploadFileTypes: allowedFileTypes.join(',')
78+
allowedUploadFileTypes: allowedFileTypes.join(','),
79+
pdfFile: {
80+
usage: pdfUsage,
81+
legacyBuild: pdfLegacyBuild
82+
}
6783
}
6884
chatbotConfig.fullFileUpload = value
6985

@@ -120,6 +136,14 @@ const FileUpload = ({ dialogProps }) => {
120136
const allowedFileTypes = chatbotConfig.fullFileUpload.allowedUploadFileTypes.split(',')
121137
setAllowedFileTypes(allowedFileTypes)
122138
}
139+
if (chatbotConfig.fullFileUpload?.pdfFile) {
140+
if (chatbotConfig.fullFileUpload.pdfFile.usage) {
141+
setPdfUsage(chatbotConfig.fullFileUpload.pdfFile.usage)
142+
}
143+
if (chatbotConfig.fullFileUpload.pdfFile.legacyBuild !== undefined) {
144+
setPdfLegacyBuild(chatbotConfig.fullFileUpload.pdfFile.legacyBuild)
145+
}
146+
}
123147
} catch (e) {
124148
setChatbotConfig({})
125149
}
@@ -202,6 +226,26 @@ const FileUpload = ({ dialogProps }) => {
202226
</div>
203227
))}
204228
</div>
229+
230+
<Box sx={{ marginBottom: 3 }}>
231+
<Typography sx={{ fontSize: 14, fontWeight: 500, marginBottom: 1 }}>PDF Usage</Typography>
232+
<FormControl disabled={!fullFileUpload}>
233+
<RadioGroup name='pdf-usage' value={pdfUsage} onChange={handlePdfUsageChange}>
234+
<FormControlLabel value='perPage' control={<Radio />} label='One document per page' />
235+
<FormControlLabel value='perFile' control={<Radio />} label='One document per file' />
236+
</RadioGroup>
237+
</FormControl>
238+
</Box>
239+
240+
<Box sx={{ marginBottom: 3 }}>
241+
<SwitchInput
242+
label='Use Legacy Build (for PDF compatibility issues)'
243+
onChange={handleLegacyBuildChange}
244+
value={pdfLegacyBuild}
245+
disabled={!fullFileUpload}
246+
/>
247+
</Box>
248+
205249
<StyledButton style={{ marginBottom: 10, marginTop: 20 }} variant='contained' onClick={onSave}>
206250
Save
207251
</StyledButton>

0 commit comments

Comments
 (0)