diff --git a/package.json b/package.json index c6fc5917..5ba4df89 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ }, "dependencies": { "hightable": "0.15.6", - "hyparquet": "1.13.3", + "hyparquet": "1.13.5", "hyparquet-compressors": "1.1.1", "icebird": "0.3.0", "react": "18.3.1", @@ -69,11 +69,11 @@ "@storybook/react-vite": "8.6.14", "@storybook/test": "8.6.14", "@testing-library/react": "16.3.0", - "@types/node": "22.15.18", - "@types/react": "19.1.4", + "@types/node": "22.15.21", + "@types/react": "19.1.5", "@types/react-dom": "19.1.5", "@vitejs/plugin-react": "4.4.1", - "@vitest/coverage-v8": "3.1.3", + "@vitest/coverage-v8": "3.1.4", "eslint": "9.27.0", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "5.2.0", @@ -87,7 +87,7 @@ "typescript": "5.8.3", "typescript-eslint": "8.32.1", "vite": "6.3.5", - "vitest": "3.1.3" + "vitest": "3.1.4" }, "eslintConfig": { "extends": [ diff --git a/src/lib/sources/httpSource.ts b/src/lib/sources/httpSource.ts index 132a0d67..545adf06 100644 --- a/src/lib/sources/httpSource.ts +++ b/src/lib/sources/httpSource.ts @@ -1,55 +1,53 @@ import { DirSource, FileMetadata, FileSource, SourcePart } from './types.js' import { getFileName } from './utils.js' -function s3list(bucket: string, prefix: string) { +async function s3list(bucket: string, prefix: string) { const url = `https://${bucket}.s3.amazonaws.com/?list-type=2&prefix=${prefix}&delimiter=/` - return fetch(url) - .then(res => { - if (!res.ok) throw new Error(`${res.status} ${res.statusText}`) - return res.text() - }) - .then(text => { - const results = [] + const result = await fetch(url) + if (!result.ok) { + throw new Error(`${result.status} ${result.statusText}`) + } + const text = await result.text() + const results = [] - // Parse regular objects (files and explicit directories) - const contentsRegex = /(.*?)<\/Contents>/gs - const contentsMatches = text.match(contentsRegex) ?? [] + // Parse regular objects (files and explicit directories) + const contentsRegex = /(.*?)<\/Contents>/gs + const contentsMatches = text.match(contentsRegex) ?? [] - for (const match of contentsMatches) { - const keyMatch = /(.*?)<\/Key>/.exec(match) - const lastModifiedMatch = /(.*?)<\/LastModified>/.exec(match) - const sizeMatch = /(.*?)<\/Size>/.exec(match) - const eTagMatch = /"(.*?)"<\/ETag>/.exec(match) ?? /"(.*?)"<\/ETag>/.exec(match) + for (const match of contentsMatches) { + const keyMatch = /(.*?)<\/Key>/.exec(match) + const lastModifiedMatch = /(.*?)<\/LastModified>/.exec(match) + const sizeMatch = /(.*?)<\/Size>/.exec(match) + const eTagMatch = /"(.*?)"<\/ETag>/.exec(match) ?? /"(.*?)"<\/ETag>/.exec(match) - if (!keyMatch || !lastModifiedMatch) continue + if (!keyMatch || !lastModifiedMatch) continue - const key = keyMatch[1] - const lastModified = lastModifiedMatch[1] - const size = sizeMatch ? parseInt(sizeMatch[1] ?? '', 10) : undefined - const eTag = eTagMatch ? eTagMatch[1] : undefined + const key = keyMatch[1] + const lastModified = lastModifiedMatch[1] + const size = sizeMatch ? parseInt(sizeMatch[1] ?? '', 10) : undefined + const eTag = eTagMatch ? eTagMatch[1] : undefined - results.push({ key, lastModified, size, eTag }) - } + results.push({ key, lastModified, size, eTag }) + } - // Parse CommonPrefixes (virtual directories) - const prefixRegex = /(.*?)<\/CommonPrefixes>/gs - const prefixMatches = text.match(prefixRegex) ?? [] - - for (const match of prefixMatches) { - const prefixMatch = /(.*?)<\/Prefix>/.exec(match) - if (!prefixMatch) continue - - const key = prefixMatch[1] - results.push({ - key, - lastModified: new Date().toISOString(), // No lastModified for CommonPrefixes - size: 0, - isCommonPrefix: true, - }) - } + // Parse CommonPrefixes (virtual directories) + const prefixRegex = /(.*?)<\/CommonPrefixes>/gs + const prefixMatches = text.match(prefixRegex) ?? [] + + for (const match of prefixMatches) { + const prefixMatch = /(.*?)<\/Prefix>/.exec(match) + if (!prefixMatch) continue - return results + const key = prefixMatch[1] + results.push({ + key, + lastModified: new Date().toISOString(), // No lastModified for CommonPrefixes + size: 0, + isCommonPrefix: true, }) + } + + return results } function getSourceParts(sourceId: string): SourcePart[] {