From fbc0fb4962fc7aef5508437f94c97f9c69becd7a Mon Sep 17 00:00:00 2001 From: hoophalab <200652805+hoophalab@users.noreply.github.com> Date: Tue, 22 Jul 2025 14:22:29 -0400 Subject: [PATCH 01/15] feat: Add presto search route. --- .../webui/client/src/api/presto-search.ts | 43 +++++++++ .../SearchControls/Presto/RunButton/index.tsx | 15 ++++ components/webui/server/package-lock.json | 38 ++++++++ components/webui/server/package.json | 4 +- components/webui/server/settings.json | 6 +- .../src/routes/api/presto-search/index.ts | 89 +++++++++++++++++++ .../webui/server/src/schemas/presto-search.ts | 20 +++++ 7 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 components/webui/client/src/api/presto-search.ts create mode 100644 components/webui/server/src/routes/api/presto-search/index.ts create mode 100644 components/webui/server/src/schemas/presto-search.ts diff --git a/components/webui/client/src/api/presto-search.ts b/components/webui/client/src/api/presto-search.ts new file mode 100644 index 0000000000..876ccd8179 --- /dev/null +++ b/components/webui/client/src/api/presto-search.ts @@ -0,0 +1,43 @@ +import { + Static, + Type, +} from "@sinclair/typebox"; +import {Value} from "@sinclair/typebox/value"; +import axios from "axios"; + + +// eslint-disable-next-line no-warning-comments +// TODO: Replace with shared type from the `@common` directory once refactoring is completed. +// Currently, server schema types require typebox dependency so they cannot be moved to the +// `@common` directory with current implementation. +const StringSchema = Type.String({ + minLength: 1, +}); + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +const PrestoSearchJobCreationSchema = Type.Object({ + queryString: StringSchema, +}); + +type PrestoSearchJobCreation = Static; + +const PrestoJobSchema = Type.Object( + { + searchJobId: StringSchema, + } +); + +type PrestoJob = Static; + +/** + * Sends post request to server tosubmit presto query. + * + * @param payload + * @return + */ +const submitQuery = async (payload: PrestoSearchJobCreation): Promise => { + const ret = await axios.post("/api/presto-search/query", payload); + return Value.Parse(PrestoJobSchema, ret.data); +}; + +export {submitQuery}; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx index 839d96209f..51b7c40ab8 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx @@ -1,9 +1,12 @@ +import {useCallback} from "react"; + import {CaretRightOutlined} from "@ant-design/icons"; import { Button, Tooltip, } from "antd"; +import {submitQuery} from "../../../../../api/presto-search"; import useSearchStore from "../../../SearchState/index"; @@ -20,6 +23,17 @@ const RunButton = () => { "Enter SQL query to run" : ""; + const handleClick = useCallback(() => { + submitQuery({queryString}) + .then(({searchJobId}) => { + const {updateSearchJobId} = useSearchStore.getState(); + updateSearchJobId(searchJobId); + }) + .catch((err: unknown) => { + console.error("Failed to submit query:", err); + }); + }, [queryString]); + return ( diff --git a/components/webui/server/package-lock.json b/components/webui/server/package-lock.json index 976f56548c..97a013757b 100644 --- a/components/webui/server/package-lock.json +++ b/components/webui/server/package-lock.json @@ -29,10 +29,12 @@ "fastify-plugin": "^5.0.1", "http-status-codes": "^2.3.0", "pino-pretty": "^13.0.0", + "presto-client": "^1.1.0", "socket.io": "^4.8.1", "typescript": "~5.7.3" }, "devDependencies": { + "@types/presto-client": "^1.0.2", "concurrently": "^9.1.2", "eslint-config-yscope": "latest", "fastify-cli": "^7.4.0", @@ -3674,6 +3676,13 @@ "undici-types": "~6.20.0" } }, + "node_modules/@types/presto-client": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/presto-client/-/presto-client-1.0.2.tgz", + "integrity": "sha512-1q7QtX+ykoX5zjWTJpntIzxyhr20ZHrEySk2p2PQCOwPI7DJtGsL62KKFZMsTqPjikpjLL71BWtVA0cPMyYdfA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -7041,6 +7050,26 @@ "license": "ISC", "peer": true }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -10065,6 +10094,15 @@ "node": ">= 0.8.0" } }, + "node_modules/presto-client": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/presto-client/-/presto-client-1.1.0.tgz", + "integrity": "sha512-DOWEKp0eHP/x6Fupk5673vZND7OUxFtV9VUO9HMvf4DFzoWKTLMRAJ3o5/7Mgs5z9w5BEUKU88IZaume6LMelw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.3" + } + }, "node_modules/prismjs": { "version": "1.29.0", "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", diff --git a/components/webui/server/package.json b/components/webui/server/package.json index b2bd7bafa6..ebe68d80ec 100644 --- a/components/webui/server/package.json +++ b/components/webui/server/package.json @@ -22,10 +22,10 @@ "@aws-sdk/s3-request-presigner": "^3.758.0", "@fastify/autoload": "^6.3.0", "@fastify/env": "^5.0.2", + "@fastify/http-proxy": "^11.3.0", "@fastify/mongodb": "^9.0.2", "@fastify/mysql": "^5.0.2", "@fastify/rate-limit": "^10.2.2", - "@fastify/http-proxy": "^11.3.0", "@fastify/sensible": "^6.0.3", "@fastify/static": "^8.1.1", "@fastify/type-provider-typebox": "^5.1.0", @@ -38,10 +38,12 @@ "fastify-plugin": "^5.0.1", "http-status-codes": "^2.3.0", "pino-pretty": "^13.0.0", + "presto-client": "^1.1.0", "socket.io": "^4.8.1", "typescript": "~5.7.3" }, "devDependencies": { + "@types/presto-client": "^1.0.2", "concurrently": "^9.1.2", "eslint-config-yscope": "latest", "fastify-cli": "^7.4.0", diff --git a/components/webui/server/settings.json b/components/webui/server/settings.json index 5f7f789364..2727c573d3 100644 --- a/components/webui/server/settings.json +++ b/components/webui/server/settings.json @@ -16,5 +16,9 @@ "StreamTargetUncompressedSize": 134217728, "StreamFilesS3Region": null, "StreamFilesS3PathPrefix": null, - "StreamFilesS3Profile": null + "StreamFilesS3Profile": null, + + "EnablePresto": true, + "PrestoHost": "localhost", + "PrestoPort": 8080 } diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts new file mode 100644 index 0000000000..7d58e9d648 --- /dev/null +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -0,0 +1,89 @@ +import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; +import {StatusCodes} from "http-status-codes"; +import {Client} from "presto-client"; + +import settings from "../../../../settings.json" with {type: "json"}; +import {ErrorSchema} from "../../../schemas/error.js"; +import { + PrestoJobSchema, + PrestoSearchJobCreationSchema, +} from "../../../schemas/presto-search.js"; +import {Nullable} from "../../../typings/common.js"; + + +/** + * Presto search API routes. + * + * @param fastify + */ +const plugin: FastifyPluginAsyncTypebox = async (fastify) => { + if (false === settings.EnablePresto) { + return; + } + + const client = new Client({host: settings.PrestoHost, port: settings.PrestoPort}); + + /** + * Submits a search query and initiates the search process. + */ + fastify.post( + "/query", + { + schema: { + body: PrestoSearchJobCreationSchema, + response: { + [StatusCodes.CREATED]: PrestoJobSchema, + [StatusCodes.INTERNAL_SERVER_ERROR]: ErrorSchema, + }, + tags: ["Search"], + }, + }, + + async (request, reply) => { + const {queryString} = request.body; + + let searchJobId: Nullable = null; + + searchJobId = await new Promise((resolve) => { + client.execute({ + data: (error, data, columns) => { + if (null === searchJobId) { + request.log.error("Data arrived from Presto before the search job id."); + } + request.log.info({error, searchJobId, columns, data}, "Presto data"); + }, + error: (error) => { + request.log.info(error, "Presto search failed"); + + // The `/query` endpoint will catch this error if `resolve` hasn't + // been called. + throw new Error("Presto search failed."); + }, + query: queryString, + state: (error, newSearchJobId, stats) => { + request.log.info({ + error: error, + searchJobId: newSearchJobId, + state: stats.state, + }, "Presto search state updated"); + + resolve(newSearchJobId); + }, + success: () => { + request.log.info("Presto search succeeded"); + }, + }); + }); + + reply.code(StatusCodes.CREATED); + + if (null === searchJobId) { + throw new Error("searchJobId is null"); + } + + return {searchJobId}; + } + ); +}; + +export default plugin; diff --git a/components/webui/server/src/schemas/presto-search.ts b/components/webui/server/src/schemas/presto-search.ts new file mode 100644 index 0000000000..0d32f088b1 --- /dev/null +++ b/components/webui/server/src/schemas/presto-search.ts @@ -0,0 +1,20 @@ +import {Type} from "@sinclair/typebox"; + +import {StringSchema} from "./common.js"; + + +/** + * Schema for request to create a new query job. + */ +const PrestoSearchJobCreationSchema = Type.Object({ + queryString: StringSchema, +}); + +const PrestoJobSchema = Type.Object({ + searchJobId: StringSchema, +}); + +export { + PrestoJobSchema, + PrestoSearchJobCreationSchema, +}; From 73d41d5e52341208dab65d6d007947028c7575b2 Mon Sep 17 00:00:00 2001 From: hoophalab <200652805+hoophalab@users.noreply.github.com> Date: Fri, 25 Jul 2025 17:06:07 -0400 Subject: [PATCH 02/15] fix a typo in jsdoc --- components/webui/client/src/api/presto-search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/webui/client/src/api/presto-search.ts b/components/webui/client/src/api/presto-search.ts index 876ccd8179..41705a12ad 100644 --- a/components/webui/client/src/api/presto-search.ts +++ b/components/webui/client/src/api/presto-search.ts @@ -30,7 +30,7 @@ const PrestoJobSchema = Type.Object( type PrestoJob = Static; /** - * Sends post request to server tosubmit presto query. + * Sends post request to server to submit presto query. * * @param payload * @return From 879941e0232bb7e1e708df87be5c2838385158c9 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 31 Jul 2025 18:58:45 +0000 Subject: [PATCH 03/15] latest --- components/webui/client/public/settings.json | 2 +- .../webui/client/src/api/presto-search.ts | 43 --------- .../SearchControls/Presto/RunButton/index.tsx | 11 +-- .../Presto/presto-search-requests.ts | 26 ++++++ components/webui/server/settings.json | 2 +- .../webui/server/src/plugins/app/Presto.ts | 47 ++++++++++ .../src/routes/api/presto-search/index.ts | 90 ++++++++++--------- .../webui/server/src/schemas/presto-search.ts | 13 +-- 8 files changed, 131 insertions(+), 103 deletions(-) delete mode 100644 components/webui/client/src/api/presto-search.ts create mode 100644 components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts create mode 100644 components/webui/server/src/plugins/app/Presto.ts diff --git a/components/webui/client/public/settings.json b/components/webui/client/public/settings.json index 67f60e795b..05048049af 100644 --- a/components/webui/client/public/settings.json +++ b/components/webui/client/public/settings.json @@ -1,6 +1,6 @@ { "ClpStorageEngine": "clp", - "ClpQueryEngine": "native", + "ClpQueryEngine": "presto", "MongoDbSearchResultsMetadataCollectionName": "results-metadata", "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", diff --git a/components/webui/client/src/api/presto-search.ts b/components/webui/client/src/api/presto-search.ts deleted file mode 100644 index 41705a12ad..0000000000 --- a/components/webui/client/src/api/presto-search.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { - Static, - Type, -} from "@sinclair/typebox"; -import {Value} from "@sinclair/typebox/value"; -import axios from "axios"; - - -// eslint-disable-next-line no-warning-comments -// TODO: Replace with shared type from the `@common` directory once refactoring is completed. -// Currently, server schema types require typebox dependency so they cannot be moved to the -// `@common` directory with current implementation. -const StringSchema = Type.String({ - minLength: 1, -}); - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const PrestoSearchJobCreationSchema = Type.Object({ - queryString: StringSchema, -}); - -type PrestoSearchJobCreation = Static; - -const PrestoJobSchema = Type.Object( - { - searchJobId: StringSchema, - } -); - -type PrestoJob = Static; - -/** - * Sends post request to server to submit presto query. - * - * @param payload - * @return - */ -const submitQuery = async (payload: PrestoSearchJobCreation): Promise => { - const ret = await axios.post("/api/presto-search/query", payload); - return Value.Parse(PrestoJobSchema, ret.data); -}; - -export {submitQuery}; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx index 51b7c40ab8..e0f5c26d48 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/RunButton/index.tsx @@ -6,8 +6,8 @@ import { Tooltip, } from "antd"; -import {submitQuery} from "../../../../../api/presto-search"; import useSearchStore from "../../../SearchState/index"; +import {handlePrestoQuerySubmit} from "../presto-search-requests"; /** @@ -24,14 +24,7 @@ const RunButton = () => { ""; const handleClick = useCallback(() => { - submitQuery({queryString}) - .then(({searchJobId}) => { - const {updateSearchJobId} = useSearchStore.getState(); - updateSearchJobId(searchJobId); - }) - .catch((err: unknown) => { - console.error("Failed to submit query:", err); - }); + handlePrestoQuerySubmit({queryString}); }, [queryString]); return ( diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts new file mode 100644 index 0000000000..f6bcff6a80 --- /dev/null +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts @@ -0,0 +1,26 @@ +import {submitQuery} from "../../../../api/presto-search"; +import type {PrestoQueryJobCreationSchema} from "../../../../api/presto-search"; + +/** + * Submits a new Presto query to server. + * + * @param payload + */ +const handlePrestoQuerySubmit = (payload: PrestoQueryJobCreationSchema) => { + submitQuery(payload) + .then((result) => { + const {searchJobId} = result.data; + console.debug( + "Presto search job created - ", + "Search job ID:", + searchJobId + ); + }) + .catch((err: unknown) => { + console.error("Failed to submit query:", err); + }); +}; + +export { + handlePrestoQuerySubmit, +}; diff --git a/components/webui/server/settings.json b/components/webui/server/settings.json index 2727c573d3..a5bb5c8d0d 100644 --- a/components/webui/server/settings.json +++ b/components/webui/server/settings.json @@ -18,7 +18,7 @@ "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, - "EnablePresto": true, + "ClpQueryEngine": "presto", "PrestoHost": "localhost", "PrestoPort": 8080 } diff --git a/components/webui/server/src/plugins/app/Presto.ts b/components/webui/server/src/plugins/app/Presto.ts new file mode 100644 index 0000000000..dd6ccf31be --- /dev/null +++ b/components/webui/server/src/plugins/app/Presto.ts @@ -0,0 +1,47 @@ +import fp from "fastify-plugin"; +import { + Client, + ClientOptions, +} from "presto-client"; + +import settings from "../../../settings.json" with {type: "json"}; + + +/** + * Class to manage Presto client connections. + */ +class Presto { + readonly client; + + /** + * @param clientOptions + */ + constructor (clientOptions: ClientOptions) { + this.client = new Client(clientOptions); + } +} + +declare module "fastify" { + interface FastifyInstance { + Presto?: Presto; + } +} + +export default fp( + (fastify) => { + if ("presto" !== settings.ClpQueryEngine) { + return; + } + + const clientOptions: ClientOptions = { + host: settings.PrestoHost, + port: settings.PrestoPort, + }; + + fastify.log.info( + clientOptions, + "Initializing Presto" + ); + fastify.decorate("Presto", new Presto(clientOptions)); + }, +); diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 7d58e9d648..858b91b496 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -1,14 +1,11 @@ import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; import {StatusCodes} from "http-status-codes"; -import {Client} from "presto-client"; -import settings from "../../../../settings.json" with {type: "json"}; import {ErrorSchema} from "../../../schemas/error.js"; import { - PrestoJobSchema, - PrestoSearchJobCreationSchema, + PrestoQueryJobCreationSchema, + PrestoQueryJobSchema, } from "../../../schemas/presto-search.js"; -import {Nullable} from "../../../typings/common.js"; /** @@ -17,11 +14,11 @@ import {Nullable} from "../../../typings/common.js"; * @param fastify */ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { - if (false === settings.EnablePresto) { - return; - } + const {Presto} = fastify; - const client = new Client({host: settings.PrestoHost, port: settings.PrestoPort}); + if ("undefined" === typeof Presto) { + throw new Error("Presto not available"); + } /** * Submits a search query and initiates the search process. @@ -30,57 +27,62 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { "/query", { schema: { - body: PrestoSearchJobCreationSchema, + body: PrestoQueryJobCreationSchema, response: { - [StatusCodes.CREATED]: PrestoJobSchema, + [StatusCodes.CREATED]: PrestoQueryJobSchema, [StatusCodes.INTERNAL_SERVER_ERROR]: ErrorSchema, }, - tags: ["Search"], + tags: ["Presto Search"], }, }, async (request, reply) => { const {queryString} = request.body; - let searchJobId: Nullable = null; + let searchJobId: string; - searchJobId = await new Promise((resolve) => { - client.execute({ - data: (error, data, columns) => { - if (null === searchJobId) { - request.log.error("Data arrived from Presto before the search job id."); - } - request.log.info({error, searchJobId, columns, data}, "Presto data"); - }, - error: (error) => { - request.log.info(error, "Presto search failed"); + try { + searchJobId = await new Promise((resolve, reject) => { + let isResolved = false; - // The `/query` endpoint will catch this error if `resolve` hasn't - // been called. - throw new Error("Presto search failed."); - }, - query: queryString, - state: (error, newSearchJobId, stats) => { - request.log.info({ - error: error, - searchJobId: newSearchJobId, - state: stats.state, - }, "Presto search state updated"); + Presto.client.execute({ + // eslint-disable-next-line no-warning-comments + // TODO: Data, error, and success handlers are dummy implementations + // and should be completed. + data: (_, data, columns) => { + request.log.info({columns, data}, "Presto data"); + }, + error: (error) => { + request.log.info(error, "Presto search failed"); + if (false === isResolved) { + isResolved = true; + reject(new Error("Presto search failed")); + } + }, + query: queryString, + state: (_, queryId, stats) => { + request.log.info({ + searchJobId: queryId, + state: stats.state, + }, "Presto search state updated"); - resolve(newSearchJobId); - }, - success: () => { - request.log.info("Presto search succeeded"); - }, + if (false === isResolved) { + isResolved = true; + resolve(queryId); + } + }, + success: () => { + request.log.info("Presto search succeeded"); + }, + }); }); - }); + } catch (error) { + request.log.error(error, "Failed to submit Presto query"); + throw error; + } reply.code(StatusCodes.CREATED); - if (null === searchJobId) { - throw new Error("searchJobId is null"); - } - return {searchJobId}; } ); diff --git a/components/webui/server/src/schemas/presto-search.ts b/components/webui/server/src/schemas/presto-search.ts index 0d32f088b1..ec4a92002d 100644 --- a/components/webui/server/src/schemas/presto-search.ts +++ b/components/webui/server/src/schemas/presto-search.ts @@ -4,17 +4,20 @@ import {StringSchema} from "./common.js"; /** - * Schema for request to create a new query job. + * Schema for request to create a new Presto query job. */ -const PrestoSearchJobCreationSchema = Type.Object({ +const PrestoQueryJobCreationSchema = Type.Object({ queryString: StringSchema, }); -const PrestoJobSchema = Type.Object({ +/** + * Schema to identify a Presto query job. + */ +const PrestoQueryJobSchema = Type.Object({ searchJobId: StringSchema, }); export { - PrestoJobSchema, - PrestoSearchJobCreationSchema, + PrestoQueryJobCreationSchema, + PrestoQueryJobSchema, }; From b2c8614f0f4d2957b6a9f6da9cc83976ebc3991f Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 31 Jul 2025 19:32:19 +0000 Subject: [PATCH 04/15] latest --- components/webui/client/public/settings.json | 2 +- .../client/src/api/presto-search/index.ts | 36 +++++++++++++++++++ .../Presto/presto-search-requests.ts | 11 +++--- components/webui/server/settings.json | 4 +-- .../src/routes/api/presto-search/index.ts | 7 ++-- 5 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 components/webui/client/src/api/presto-search/index.ts diff --git a/components/webui/client/public/settings.json b/components/webui/client/public/settings.json index 05048049af..67f60e795b 100644 --- a/components/webui/client/public/settings.json +++ b/components/webui/client/public/settings.json @@ -1,6 +1,6 @@ { "ClpStorageEngine": "clp", - "ClpQueryEngine": "presto", + "ClpQueryEngine": "native", "MongoDbSearchResultsMetadataCollectionName": "results-metadata", "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", diff --git a/components/webui/client/src/api/presto-search/index.ts b/components/webui/client/src/api/presto-search/index.ts new file mode 100644 index 0000000000..fdd34b6aa1 --- /dev/null +++ b/components/webui/client/src/api/presto-search/index.ts @@ -0,0 +1,36 @@ +import axios, {AxiosResponse} from "axios"; + + +// eslint-disable-next-line no-warning-comments +// TODO: Replace with shared type from the `@common` directory once refactoring is completed. +// Currently, server schema types require typebox dependency so they cannot be moved to the +// `@common` directory with current implementation. +type PrestoQueryJobCreationSchema = { + queryString: string; +}; + +type PrestoQueryJobSchema = { + searchJobId: string; +}; + + +/** + * Sends post request to server to submit presto query. + * + * @param payload + * @return + */ +const submitQuery = async ( + payload: PrestoQueryJobCreationSchema +): Promise> => { + console.log("Submitting query:", JSON.stringify(payload)); + + return axios.post("/api/presto-search/query", payload); +}; + +export type { + PrestoQueryJobCreationSchema, + PrestoQueryJobSchema, +}; + +export {submitQuery}; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts index f6bcff6a80..a42e4110d1 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Presto/presto-search-requests.ts @@ -1,5 +1,8 @@ -import {submitQuery} from "../../../../api/presto-search"; -import type {PrestoQueryJobCreationSchema} from "../../../../api/presto-search"; +import { + type PrestoQueryJobCreationSchema, + submitQuery, +} from "../../../../api/presto-search"; + /** * Submits a new Presto query to server. @@ -21,6 +24,4 @@ const handlePrestoQuerySubmit = (payload: PrestoQueryJobCreationSchema) => { }); }; -export { - handlePrestoQuerySubmit, -}; +export {handlePrestoQuerySubmit}; diff --git a/components/webui/server/settings.json b/components/webui/server/settings.json index a5bb5c8d0d..4a5990b32f 100644 --- a/components/webui/server/settings.json +++ b/components/webui/server/settings.json @@ -18,7 +18,7 @@ "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, - "ClpQueryEngine": "presto", + "ClpQueryEngine": "native", "PrestoHost": "localhost", - "PrestoPort": 8080 + "PrestoPort": 8889 } diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 858b91b496..2c8e8acc23 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -17,11 +17,12 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const {Presto} = fastify; if ("undefined" === typeof Presto) { - throw new Error("Presto not available"); + // If Presto client is not available, skip the plugin registration. + return; } /** - * Submits a search query and initiates the search process. + * Submits a search query. */ fastify.post( "/query", @@ -48,7 +49,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { Presto.client.execute({ // eslint-disable-next-line no-warning-comments // TODO: Data, error, and success handlers are dummy implementations - // and should be completed. + // and will be replaced with proper implementations. data: (_, data, columns) => { request.log.info({columns, data}, "Presto data"); }, From d1ce3f3a378ba68164ccbbbbd4c21dc0692cdc28 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 31 Jul 2025 20:51:46 +0000 Subject: [PATCH 05/15] latest --- .../src/routes/api/presto-search/index.ts | 77 +++++++++++++++++-- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 2c8e8acc23..a8a0e6d775 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -1,6 +1,11 @@ import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; import {StatusCodes} from "http-status-codes"; +import { + SEARCH_SIGNAL, + type SearchResultsMetadataDocument, +} from "../../../../../common/index.js"; +import settings from "../../../../settings.json" with {type: "json"}; import {ErrorSchema} from "../../../schemas/error.js"; import { PrestoQueryJobCreationSchema, @@ -14,13 +19,22 @@ import { * @param fastify */ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { - const {Presto} = fastify; + const {Presto, mongo} = fastify; if ("undefined" === typeof Presto) { // If Presto client is not available, skip the plugin registration. return; } + if ("undefined" === typeof mongo) { + throw new Error("MongoDB not found"); + } + + const mongoDb = mongo.db; + const searchResultsMetadataCollection = mongoDb.collection( + settings.MongoDbSearchResultsMetadataCollectionName + ); + /** * Submits a search query. */ @@ -45,18 +59,45 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { try { searchJobId = await new Promise((resolve, reject) => { let isResolved = false; + let resultCount = 0; Presto.client.execute({ - // eslint-disable-next-line no-warning-comments - // TODO: Data, error, and success handlers are dummy implementations - // and will be replaced with proper implementations. data: (_, data, columns) => { - request.log.info({columns, data}, "Presto data"); + request.log.info({columns, dataLength: data.length}, "Presto data received"); + + // Store data in MongoDB collection for this search job + if (data && data.length > 0) { + const collection = mongoDb.collection(searchJobId); + const documents = data.map((row) => ({ + _id: resultCount++, + timestamp: Date.now(), + message: row, + columns: columns, + })); + + collection.insertMany(documents).catch((err: unknown) => { + request.log.error(err, "Failed to insert Presto data into MongoDB"); + }); + } }, error: (error) => { - request.log.info(error, "Presto search failed"); + request.log.error(error, "Presto search failed"); if (false === isResolved) { isResolved = true; + + // Update metadata with error + searchResultsMetadataCollection.updateOne( + {_id: searchJobId}, + { + $set: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: String(error), + } + } + ).catch((err: unknown) => { + request.log.error(err, "Failed to update error metadata"); + }); + reject(new Error("Presto search failed")); } }, @@ -74,9 +115,33 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { }, success: () => { request.log.info("Presto search succeeded"); + + // Update metadata on success + searchResultsMetadataCollection.updateOne( + {_id: searchJobId}, + { + $set: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: null, + } + } + ).catch((err: unknown) => { + request.log.error(err, "Failed to update success metadata"); + }); }, }); }); + + // Create MongoDB collection for storing query results + await mongoDb.createCollection(searchJobId); + + // Insert initial metadata + await searchResultsMetadataCollection.insertOne({ + _id: searchJobId, + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + errorMsg: null, + }); + } catch (error) { request.log.error(error, "Failed to submit Presto query"); throw error; From 20b4f2fb8f984806046971cb9a11ae2e3500d221 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 1 Aug 2025 18:54:43 +0000 Subject: [PATCH 06/15] latest: --- .../src/routes/api/presto-search/index.ts | 84 ++++--------------- .../src/routes/api/presto-search/typings.ts | 17 ++++ .../src/routes/api/presto-search/utils.ts | 59 +++++++++++++ 3 files changed, 93 insertions(+), 67 deletions(-) create mode 100644 components/webui/server/src/routes/api/presto-search/typings.ts create mode 100644 components/webui/server/src/routes/api/presto-search/utils.ts diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index a8a0e6d775..d3b5681c5f 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -1,16 +1,12 @@ import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; import {StatusCodes} from "http-status-codes"; -import { - SEARCH_SIGNAL, - type SearchResultsMetadataDocument, -} from "../../../../../common/index.js"; -import settings from "../../../../settings.json" with {type: "json"}; import {ErrorSchema} from "../../../schemas/error.js"; import { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, } from "../../../schemas/presto-search.js"; +import {insertPrestoRowsToMongo} from "./utils.js"; /** @@ -18,23 +14,20 @@ import { * * @param fastify */ +// eslint-disable-next-line max-lines-per-function const plugin: FastifyPluginAsyncTypebox = async (fastify) => { const {Presto, mongo} = fastify; + const mongoDb = mongo.db; if ("undefined" === typeof Presto) { // If Presto client is not available, skip the plugin registration. return; } - if ("undefined" === typeof mongo) { - throw new Error("MongoDB not found"); + if ("undefined" === typeof mongoDb) { + throw new Error("MongoDB database not found"); } - const mongoDb = mongo.db; - const searchResultsMetadataCollection = mongoDb.collection( - settings.MongoDbSearchResultsMetadataCollectionName - ); - /** * Submits a search query. */ @@ -59,45 +52,26 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { try { searchJobId = await new Promise((resolve, reject) => { let isResolved = false; - let resultCount = 0; Presto.client.execute({ data: (_, data, columns) => { - request.log.info({columns, dataLength: data.length}, "Presto data received"); - - // Store data in MongoDB collection for this search job - if (data && data.length > 0) { - const collection = mongoDb.collection(searchJobId); - const documents = data.map((row) => ({ - _id: resultCount++, - timestamp: Date.now(), - message: row, - columns: columns, - })); - - collection.insertMany(documents).catch((err: unknown) => { - request.log.error(err, "Failed to insert Presto data into MongoDB"); - }); - } + request.log.info( + `Received ${data.length} rows from Presto query` + ); + + insertPrestoRowsToMongo({ + columns: columns, + data: data, + isResolved: isResolved, + log: request.log, + mongoDb: mongoDb, + searchJobId: searchJobId, + }); }, error: (error) => { request.log.error(error, "Presto search failed"); if (false === isResolved) { isResolved = true; - - // Update metadata with error - searchResultsMetadataCollection.updateOne( - {_id: searchJobId}, - { - $set: { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: String(error), - } - } - ).catch((err: unknown) => { - request.log.error(err, "Failed to update error metadata"); - }); - reject(new Error("Presto search failed")); } }, @@ -115,33 +89,9 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { }, success: () => { request.log.info("Presto search succeeded"); - - // Update metadata on success - searchResultsMetadataCollection.updateOne( - {_id: searchJobId}, - { - $set: { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: null, - } - } - ).catch((err: unknown) => { - request.log.error(err, "Failed to update success metadata"); - }); }, }); }); - - // Create MongoDB collection for storing query results - await mongoDb.createCollection(searchJobId); - - // Insert initial metadata - await searchResultsMetadataCollection.insertOne({ - _id: searchJobId, - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - errorMsg: null, - }); - } catch (error) { request.log.error(error, "Failed to submit Presto query"); throw error; diff --git a/components/webui/server/src/routes/api/presto-search/typings.ts b/components/webui/server/src/routes/api/presto-search/typings.ts new file mode 100644 index 0000000000..67d5677ee5 --- /dev/null +++ b/components/webui/server/src/routes/api/presto-search/typings.ts @@ -0,0 +1,17 @@ +import type {FastifyBaseLogger} from "fastify"; +import type {Db} from "mongodb"; + + +/** + * Props for inserting Presto rows into MongoDB. + */ +interface InsertPrestoRowsToMongoProps { + data: unknown[][]; + columns: {name: string}[]; + searchJobId: string; + isResolved: boolean; + mongoDb: Db; + log: FastifyBaseLogger; +} + +export {InsertPrestoRowsToMongoProps}; diff --git a/components/webui/server/src/routes/api/presto-search/utils.ts b/components/webui/server/src/routes/api/presto-search/utils.ts new file mode 100644 index 0000000000..5682781b92 --- /dev/null +++ b/components/webui/server/src/routes/api/presto-search/utils.ts @@ -0,0 +1,59 @@ +import type {InsertPrestoRowsToMongoProps} from "./typings.js"; + + +/** + * Adds column names as keys to a Presto row array, returning an object. + * + * @param row The array of values from Presto. + * @param columns The array of column definitions, each with a `name` property. + * @return An object mapping column names to row values. + */ +const addKeysToPrestoRow = ( + row: unknown[], + columns: {name: string}[] +): Record => { + const obj: Record = {}; + columns.forEach((col, idx) => { + obj[col.name] = row[idx]; + }); + + return obj; +}; + +/** + * Inserts Presto rows into a MongoDB collection for a given search job. + * + * @param props + * @param props.data + * @param props.columns + * @param props.searchJobId + * @param props.isResolved + * @param props.mongoDb + * @param props.log + */ +const insertPrestoRowsToMongo = ({ + columns, + data, + isResolved, + log, + mongoDb, + searchJobId, +}: InsertPrestoRowsToMongoProps): void => { + if (false === isResolved) { + log.error("Presto data received before searchJobId was resolved; skipping insert."); + + return; + } + + if (0 < data.length && searchJobId) { + const collection = mongoDb.collection(searchJobId); + const resultDocs = data.map((row) => addKeysToPrestoRow(row, columns)); + collection.insertMany(resultDocs).catch((err: unknown) => { + log.error({err}, "Failed to insert Presto results into MongoDB"); + }); + } +}; + +export { + addKeysToPrestoRow, insertPrestoRowsToMongo, +}; From 51a1eaba3335f7b310784974b6577a19b0cc2003 Mon Sep 17 00:00:00 2001 From: Marco Date: Tue, 5 Aug 2025 14:34:11 +0000 Subject: [PATCH 07/15] latest: --- components/webui/server/src/routes/api/presto-search/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 72319b4796..a580fafc80 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -53,7 +53,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { searchJobId = await new Promise((resolve, reject) => { let isResolved = false; // eslint-disable-next-line no-warning-comments - // TODO: Data, error, and success handlers are dummy implementations + // TODO: Error, and success handlers are dummy implementations // and will be replaced with proper implementations. Presto.client.execute({ data: (_, data, columns) => { From 0573aab4407f4fbeee27aeda8933ed7e6f84278e Mon Sep 17 00:00:00 2001 From: Marco Date: Tue, 5 Aug 2025 14:47:14 +0000 Subject: [PATCH 08/15] latest --- .../src/routes/api/presto-search/index.ts | 6 +++--- .../src/routes/api/presto-search/typings.ts | 3 --- .../src/routes/api/presto-search/utils.ts | 19 +++++++++---------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index a580fafc80..6b0195b1e8 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -52,10 +52,10 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { try { searchJobId = await new Promise((resolve, reject) => { let isResolved = false; - // eslint-disable-next-line no-warning-comments - // TODO: Error, and success handlers are dummy implementations - // and will be replaced with proper implementations. Presto.client.execute({ + // eslint-disable-next-line no-warning-comments + // TODO: Error, and success handlers are dummy implementations + // and will be replaced with proper implementations. data: (_, data, columns) => { request.log.info( `Received ${data.length} rows from Presto query` diff --git a/components/webui/server/src/routes/api/presto-search/typings.ts b/components/webui/server/src/routes/api/presto-search/typings.ts index 67d5677ee5..f94fba8a50 100644 --- a/components/webui/server/src/routes/api/presto-search/typings.ts +++ b/components/webui/server/src/routes/api/presto-search/typings.ts @@ -2,9 +2,6 @@ import type {FastifyBaseLogger} from "fastify"; import type {Db} from "mongodb"; -/** - * Props for inserting Presto rows into MongoDB. - */ interface InsertPrestoRowsToMongoProps { data: unknown[][]; columns: {name: string}[]; diff --git a/components/webui/server/src/routes/api/presto-search/utils.ts b/components/webui/server/src/routes/api/presto-search/utils.ts index 5682781b92..635cab1823 100644 --- a/components/webui/server/src/routes/api/presto-search/utils.ts +++ b/components/webui/server/src/routes/api/presto-search/utils.ts @@ -2,13 +2,14 @@ import type {InsertPrestoRowsToMongoProps} from "./typings.js"; /** - * Adds column names as keys to a Presto row array, returning an object. + * Converts a Presto result row (array of values) into an object, using the provided column + * definitions to assign property names. * - * @param row The array of values from Presto. - * @param columns The array of column definitions, each with a `name` property. - * @return An object mapping column names to row values. + * @param row Array of values representing a single Presto result row. + * @param columns Array of column definitions, each containing a `name` property. + * @return An object mapping each column name to its corresponding value from the row. */ -const addKeysToPrestoRow = ( +const prestoRowToObject = ( row: unknown[], columns: {name: string}[] ): Record => { @@ -47,13 +48,11 @@ const insertPrestoRowsToMongo = ({ if (0 < data.length && searchJobId) { const collection = mongoDb.collection(searchJobId); - const resultDocs = data.map((row) => addKeysToPrestoRow(row, columns)); + const resultDocs = data.map((row) => prestoRowToObject(row, columns)); collection.insertMany(resultDocs).catch((err: unknown) => { - log.error({err}, "Failed to insert Presto results into MongoDB"); + log.error(err, "Failed to insert Presto results into MongoDB"); }); } }; -export { - addKeysToPrestoRow, insertPrestoRowsToMongo, -}; +export {insertPrestoRowsToMongo}; From 0953cc17672439602bdead8f69946b3e05a8dd99 Mon Sep 17 00:00:00 2001 From: Marco Date: Tue, 5 Aug 2025 19:32:44 +0000 Subject: [PATCH 09/15] metadata --- components/webui/client/src/config/index.ts | 9 +-- .../pages/SearchPage/SearchControls/index.tsx | 2 +- components/webui/common/index.ts | 32 ++++++++- components/webui/server/settings.json | 2 +- .../src/routes/api/presto-search/index.ts | 24 +++++-- .../src/routes/api/presto-search/typings.ts | 24 ++++++- .../src/routes/api/presto-search/utils.ts | 68 +++++++++++++++++-- .../server/src/routes/api/search/index.ts | 6 +- .../server/src/routes/api/search/typings.ts | 4 +- .../server/src/routes/api/search/utils.ts | 9 ++- 10 files changed, 150 insertions(+), 30 deletions(-) diff --git a/components/webui/client/src/config/index.ts b/components/webui/client/src/config/index.ts index 0fed22bb9e..8a16f8f8ea 100644 --- a/components/webui/client/src/config/index.ts +++ b/components/webui/client/src/config/index.ts @@ -1,3 +1,4 @@ +import {CLP_QUERY_ENGINES} from "../../../common/index.js"; import {settings} from "../settings"; @@ -9,14 +10,6 @@ enum CLP_STORAGE_ENGINES { CLP_S = "clp-s", } -/** - * Query engine options. - */ -enum CLP_QUERY_ENGINES { - NATIVE = "native", - PRESTO = "presto", -} - const SETTINGS_STORAGE_ENGINE = settings.ClpStorageEngine as CLP_STORAGE_ENGINES; const SETTINGS_QUERY_ENGINE = settings.ClpQueryEngine as CLP_QUERY_ENGINES; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx index b6e4c49f1c..8e8e2011b0 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/index.tsx @@ -31,7 +31,7 @@ const SearchControls = () => { return (
- {SETTINGS_QUERY_ENGINE === CLP_QUERY_ENGINES.NATIVE ? + {SETTINGS_QUERY_ENGINE !== CLP_QUERY_ENGINES.PRESTO ? ( <> {CLP_STORAGE_ENGINES.CLP_S === SETTINGS_STORAGE_ENGINE && } diff --git a/components/webui/common/index.ts b/components/webui/common/index.ts index 9045e9bfc3..ad79bca370 100644 --- a/components/webui/common/index.ts +++ b/components/webui/common/index.ts @@ -88,6 +88,33 @@ enum SEARCH_SIGNAL { RESP_QUERYING = "resp-querying", } +/** + * Presto search-related signals. + * + * This includes Presto query states and their respective signals. + */ +type PRESTO_SEARCH_SIGNAL = + | "WAITING_FOR_PREREQUISITES" + | "QUEUED" + | "WAITING_FOR_RESOURCES" + | "DISPATCHING" + | "PLANNING" + | "STARTING" + | "RUNNING" + | "FINISHING" + | "FINISHED" + | "CANCELED" + | "FAILED"; + +/** + * CLP query engines. + */ +enum CLP_QUERY_ENGINES { + CLP = "clp", + CLP_S = "clp-s", + PRESTO = "presto", +} + /** * MongoDB document for search results metadata. `numTotalResults` is optional * since it is only set when the search job is completed. @@ -98,13 +125,16 @@ interface SearchResultsMetadataDocument { // eslint-disable-next-line no-warning-comments // TODO: Replace with Nullable when the `@common` directory refactoring is completed. errorMsg: string | null; - lastSignal: SEARCH_SIGNAL; + lastSignal: SEARCH_SIGNAL | PRESTO_SEARCH_SIGNAL; numTotalResults?: number; + queryEngine: CLP_QUERY_ENGINES; } export { + CLP_QUERY_ENGINES, SEARCH_SIGNAL, }; export type { + PRESTO_SEARCH_SIGNAL, SearchResultsMetadataDocument, ClientToServerEvents, Err, diff --git a/components/webui/server/settings.json b/components/webui/server/settings.json index 4a5990b32f..9b0386021f 100644 --- a/components/webui/server/settings.json +++ b/components/webui/server/settings.json @@ -18,7 +18,7 @@ "StreamFilesS3PathPrefix": null, "StreamFilesS3Profile": null, - "ClpQueryEngine": "native", + "ClpQueryEngine": "clp", "PrestoHost": "localhost", "PrestoPort": 8889 } diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index 6b0195b1e8..4f561a19c6 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -1,12 +1,17 @@ import {FastifyPluginAsyncTypebox} from "@fastify/type-provider-typebox"; import {StatusCodes} from "http-status-codes"; +import type {SearchResultsMetadataDocument} from "../../../../../common/index.js"; +import settings from "../../../../settings.json" with {type: "json"}; import {ErrorSchema} from "../../../schemas/error.js"; import { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, } from "../../../schemas/presto-search.js"; -import {insertPrestoRowsToMongo} from "./utils.js"; +import { + insertPrestoRowsToMongo, + processPrestoStateChange, +} from "./utils.js"; /** @@ -28,6 +33,10 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { throw new Error("MongoDB database not found"); } + const searchResultsMetadataCollection = mongoDb.collection( + settings.MongoDbSearchResultsMetadataCollectionName + ); + /** * Submits a search query. */ @@ -84,10 +93,15 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { state: stats.state, }, "Presto search state updated"); - if (false === isResolved) { - isResolved = true; - resolve(queryId); - } + const logger = request.log; + isResolved = processPrestoStateChange({ + isResolved: isResolved, + logger: logger, + queryId: queryId, + resolve: resolve, + searchResultsMetadataCollection: searchResultsMetadataCollection, + state: stats.state, + }); }, success: () => { request.log.info("Presto search succeeded"); diff --git a/components/webui/server/src/routes/api/presto-search/typings.ts b/components/webui/server/src/routes/api/presto-search/typings.ts index f94fba8a50..15938023e0 100644 --- a/components/webui/server/src/routes/api/presto-search/typings.ts +++ b/components/webui/server/src/routes/api/presto-search/typings.ts @@ -1,5 +1,13 @@ import type {FastifyBaseLogger} from "fastify"; -import type {Db} from "mongodb"; +import type { + Collection, + Db, +} from "mongodb"; + +import type { + PRESTO_SEARCH_SIGNAL, + SearchResultsMetadataDocument, +} from "../../../../../common/index.js"; interface InsertPrestoRowsToMongoProps { @@ -11,4 +19,16 @@ interface InsertPrestoRowsToMongoProps { log: FastifyBaseLogger; } -export {InsertPrestoRowsToMongoProps}; +type ProcessPrestoStateChangeProps = { + queryId: string; + state: PRESTO_SEARCH_SIGNAL; + isResolved: boolean; + logger: FastifyBaseLogger; + searchResultsMetadataCollection: Collection; + resolve: (value: string) => void; +}; + +export type { + InsertPrestoRowsToMongoProps, + ProcessPrestoStateChangeProps, +}; diff --git a/components/webui/server/src/routes/api/presto-search/utils.ts b/components/webui/server/src/routes/api/presto-search/utils.ts index 635cab1823..125da7f00f 100644 --- a/components/webui/server/src/routes/api/presto-search/utils.ts +++ b/components/webui/server/src/routes/api/presto-search/utils.ts @@ -1,12 +1,16 @@ -import type {InsertPrestoRowsToMongoProps} from "./typings.js"; +import {CLP_QUERY_ENGINES} from "../../../../../common/index.js"; +import {updateSearchResultsMeta} from "../search/utils.js"; +import type { + InsertPrestoRowsToMongoProps, + ProcessPrestoStateChangeProps, +} from "./typings.js"; /** - * Converts a Presto result row (array of values) into an object, using the provided column - * definitions to assign property names. + * Converts a Presto row array to an object mapping column names to values. * - * @param row Array of values representing a single Presto result row. - * @param columns Array of column definitions, each containing a `name` property. + * @param row + * @param columns * @return An object mapping each column name to its corresponding value from the row. */ const prestoRowToObject = ( @@ -55,4 +59,56 @@ const insertPrestoRowsToMongo = ({ } }; -export {insertPrestoRowsToMongo}; +/** + * Processes Presto state changes - creates metadata on first call and resolves the promise, + * then updates metadata on subsequent calls. + * + * @param props + * @param props.queryId + * @param props.state + * @param props.isResolved + * @param props.logger + * @param props.searchResultsMetadataCollection + * @param props.resolve + * @return Updated isResolved flag + */ +const processPrestoStateChange = ({ + queryId, + state, + isResolved, + logger, + searchResultsMetadataCollection, + resolve, +}: ProcessPrestoStateChangeProps): boolean => { + // Insert metadata on first state callback + if (false === isResolved) { + searchResultsMetadataCollection.insertOne({ + _id: queryId, + lastSignal: state, + errorMsg: null, + queryEngine: CLP_QUERY_ENGINES.PRESTO, + }).catch((err: unknown) => { + logger.error(err, "Failed to insert Presto metadata"); + }); + resolve(queryId); + + return true; + } + + // Update lastSignal in metadata using Presto state names + updateSearchResultsMeta({ + fields: {lastSignal: state}, + jobId: queryId, + logger: logger, + searchResultsMetadataCollection: searchResultsMetadataCollection, + }).catch((err: unknown) => { + logger.error(err, "Failed to update Presto metadata"); + }); + + return isResolved; +}; + +export { + insertPrestoRowsToMongo, + processPrestoStateChange, +}; diff --git a/components/webui/server/src/routes/api/search/index.ts b/components/webui/server/src/routes/api/search/index.ts index a48f3737cd..7b0cd56f8b 100644 --- a/components/webui/server/src/routes/api/search/index.ts +++ b/components/webui/server/src/routes/api/search/index.ts @@ -5,6 +5,7 @@ import { import {StatusCodes} from "http-status-codes"; import { + CLP_QUERY_ENGINES, SEARCH_SIGNAL, type SearchResultsMetadataDocument, } from "../../../../../common/index.js"; @@ -44,6 +45,8 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { settings.MongoDbSearchResultsMetadataCollectionName ); + const queryEngine = settings.ClpQueryEngine as CLP_QUERY_ENGINES; + /** * Submits a search query and initiates the search process. */ @@ -113,6 +116,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { _id: searchJobId.toString(), lastSignal: SEARCH_SIGNAL.RESP_QUERYING, errorMsg: null, + queryEngine: queryEngine, }); // Defer signal update until after response is sent @@ -202,7 +206,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { lastSignal: SEARCH_SIGNAL.RESP_DONE, errorMsg: "Query cancelled before it could be completed.", }, - jobId: searchJobId, + jobId: searchJobId.toString(), lastSignal: SEARCH_SIGNAL.RESP_QUERYING, logger: request.log, searchResultsMetadataCollection: searchResultsMetadataCollection, diff --git a/components/webui/server/src/routes/api/search/typings.ts b/components/webui/server/src/routes/api/search/typings.ts index 353a8a9270..55fd671ecc 100644 --- a/components/webui/server/src/routes/api/search/typings.ts +++ b/components/webui/server/src/routes/api/search/typings.ts @@ -20,8 +20,8 @@ const SEARCH_MAX_NUM_RESULTS = 1000; type UpdateSearchResultsMetaProps = { fields: Partial; - jobId: number; - lastSignal: SEARCH_SIGNAL; + jobId: string; + lastSignal?: SEARCH_SIGNAL; logger: FastifyBaseLogger; searchResultsMetadataCollection: Collection; }; diff --git a/components/webui/server/src/routes/api/search/utils.ts b/components/webui/server/src/routes/api/search/utils.ts index 88b19b4035..e9e1b73627 100644 --- a/components/webui/server/src/routes/api/search/utils.ts +++ b/components/webui/server/src/routes/api/search/utils.ts @@ -38,11 +38,14 @@ const updateSearchResultsMeta = async ({ logger, searchResultsMetadataCollection, }: UpdateSearchResultsMetaProps) => { - const filter = { - _id: jobId.toString(), - lastSignal: lastSignal, + const filter: Record = { + _id: jobId, }; + if (lastSignal) { + filter.lastSignal = lastSignal; + } + const modifier = { $set: fields, }; From 6240164ef5b6cb8a2b047b8728618e55b7340cba Mon Sep 17 00:00:00 2001 From: Marco Date: Tue, 5 Aug 2025 20:59:22 +0000 Subject: [PATCH 10/15] latest --- .../webui/server/src/routes/api/presto-search/utils.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/components/webui/server/src/routes/api/presto-search/utils.ts b/components/webui/server/src/routes/api/presto-search/utils.ts index 125da7f00f..277476e63c 100644 --- a/components/webui/server/src/routes/api/presto-search/utils.ts +++ b/components/webui/server/src/routes/api/presto-search/utils.ts @@ -60,8 +60,9 @@ const insertPrestoRowsToMongo = ({ }; /** - * Processes Presto state changes - creates metadata on first call and resolves the promise, - * then updates metadata on subsequent calls. + * Processes Presto query state changes. On the first call, creates initial metadata document in + * MongoDB and resolves the promise which returns the queryId. On subsequent calls, updates the + * metadata document with the current state. * * @param props * @param props.queryId @@ -80,7 +81,7 @@ const processPrestoStateChange = ({ searchResultsMetadataCollection, resolve, }: ProcessPrestoStateChangeProps): boolean => { - // Insert metadata on first state callback + // Insert metadata and resolve queryId on first call if (false === isResolved) { searchResultsMetadataCollection.insertOne({ _id: queryId, @@ -95,7 +96,7 @@ const processPrestoStateChange = ({ return true; } - // Update lastSignal in metadata using Presto state names + // Update metadata on subsequent calls updateSearchResultsMeta({ fields: {lastSignal: state}, jobId: queryId, From b6c8c8b78d4a22be806c8195852c45105ea1680b Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 6 Aug 2025 18:08:40 +0000 Subject: [PATCH 11/15] latest --- components/webui/client/public/settings.json | 2 +- components/webui/common/index.ts | 2 +- .../webui/server/src/routes/api/presto-search/utils.ts | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/components/webui/client/public/settings.json b/components/webui/client/public/settings.json index 67f60e795b..b3dfa4e3e4 100644 --- a/components/webui/client/public/settings.json +++ b/components/webui/client/public/settings.json @@ -1,6 +1,6 @@ { "ClpStorageEngine": "clp", - "ClpQueryEngine": "native", + "ClpQueryEngine": "clp", "MongoDbSearchResultsMetadataCollectionName": "results-metadata", "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", diff --git a/components/webui/common/index.ts b/components/webui/common/index.ts index ad79bca370..944b5b8760 100644 --- a/components/webui/common/index.ts +++ b/components/webui/common/index.ts @@ -91,7 +91,7 @@ enum SEARCH_SIGNAL { /** * Presto search-related signals. * - * This includes Presto query states and their respective signals. + * Note: Using type instead of enum match `presto-client-node` type definition. */ type PRESTO_SEARCH_SIGNAL = | "WAITING_FOR_PREREQUISITES" diff --git a/components/webui/server/src/routes/api/presto-search/utils.ts b/components/webui/server/src/routes/api/presto-search/utils.ts index 897d310a68..4574a0aff8 100644 --- a/components/webui/server/src/routes/api/presto-search/utils.ts +++ b/components/webui/server/src/routes/api/presto-search/utils.ts @@ -5,10 +5,11 @@ import type { /** - * Converts a Presto row array to an object mapping column names to values. + * Converts a Presto result row (array of values) into an object, using the provided column + * definitions to assign property names. * - * @param row - * @param columns + * @param row Array of values representing a single Presto result row. + * @param columns Array of column definitions, each containing a `name` property. * @return An object mapping each column name to its corresponding value from the row. */ const prestoRowToObject = ( From 6fb776bfe116cb3423df9149fddf00d9e1cc69b8 Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 6 Aug 2025 18:10:06 +0000 Subject: [PATCH 12/15] latest --- components/webui/common/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/webui/common/index.ts b/components/webui/common/index.ts index 944b5b8760..55d08bdbf4 100644 --- a/components/webui/common/index.ts +++ b/components/webui/common/index.ts @@ -91,7 +91,7 @@ enum SEARCH_SIGNAL { /** * Presto search-related signals. * - * Note: Using type instead of enum match `presto-client-node` type definition. + * Note: Using type instead of enum to match `presto-client-node` type definition. */ type PRESTO_SEARCH_SIGNAL = | "WAITING_FOR_PREREQUISITES" From ff4d0c4e45a1aa968155c79ce96090794a6745bb Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 8 Aug 2025 18:04:10 +0000 Subject: [PATCH 13/15] changes --- .../src/routes/api/presto-search/index.ts | 13 ++--- .../server/src/routes/api/search/index.ts | 21 ++++---- .../server/src/routes/api/search/utils.ts | 50 +++---------------- 3 files changed, 24 insertions(+), 60 deletions(-) diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index a435b16bcb..dd19997005 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -11,7 +11,6 @@ import { PrestoQueryJobCreationSchema, PrestoQueryJobSchema, } from "../../../schemas/presto-search.js"; -import {updateSearchResultsMeta} from "../search/utils.js"; import {insertPrestoRowsToMongo} from "./utils.js"; @@ -105,6 +104,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { } }, query: queryString, + timeout: null, state: (_, queryId, stats) => { request.log.info({ searchJobId: queryId, @@ -125,13 +125,10 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { resolve(queryId); } else { // Update metadata on subsequent calls - updateSearchResultsMeta({ - fields: {lastSignal: stats.state}, - jobId: queryId, - logger: request.log, - searchResultsMetadataCollection: - searchResultsMetadataCollection, - }).catch((err: unknown) => { + searchResultsMetadataCollection.updateOne( + {_id: queryId}, + {$set: {lastSignal: stats.state}} + ).catch((err: unknown) => { request.log.error(err, "Failed to update Presto metadata"); }); } diff --git a/components/webui/server/src/routes/api/search/index.ts b/components/webui/server/src/routes/api/search/index.ts index 7b0cd56f8b..5857c449df 100644 --- a/components/webui/server/src/routes/api/search/index.ts +++ b/components/webui/server/src/routes/api/search/index.ts @@ -19,7 +19,6 @@ import {QUERY_JOB_TYPE} from "../../../typings/query.js"; import {SEARCH_MAX_NUM_RESULTS} from "./typings.js"; import { createMongoIndexes, - updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, } from "./utils.js"; @@ -201,16 +200,18 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { await QueryJobDbManager.cancelJob(searchJobId); await QueryJobDbManager.cancelJob(aggregationJobId); - await updateSearchResultsMeta({ - fields: { - lastSignal: SEARCH_SIGNAL.RESP_DONE, - errorMsg: "Query cancelled before it could be completed.", + await searchResultsMetadataCollection.updateOne( + { + _id: searchJobId.toString(), + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, }, - jobId: searchJobId.toString(), - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - logger: request.log, - searchResultsMetadataCollection: searchResultsMetadataCollection, - }); + { + $set: { + lastSignal: SEARCH_SIGNAL.RESP_DONE, + errorMsg: "Query cancelled before it could be completed.", + }, + } + ); } catch (err: unknown) { const errMsg = "Failed to submit cancel request"; request.log.error( diff --git a/components/webui/server/src/routes/api/search/utils.ts b/components/webui/server/src/routes/api/search/utils.ts index 0d19c5ac59..8df0dec1be 100644 --- a/components/webui/server/src/routes/api/search/utils.ts +++ b/components/webui/server/src/routes/api/search/utils.ts @@ -4,7 +4,6 @@ import {SEARCH_SIGNAL} from "../../../../../common/index.js"; import { CreateMongoIndexesProps, SEARCH_MAX_NUM_RESULTS, - UpdateSearchResultsMetaProps, UpdateSearchSignalWhenJobsFinishProps, } from "./typings.js"; @@ -21,39 +20,6 @@ const hasCollection = async (mongoDb: Db, collectionName: string): Promise collection.name === collectionName); }; -/** - * Modifies the search results metadata for a given job ID. - * - * @param props - * @param props.fields - * @param props.jobId - * @param props.lastSignal - * @param props.logger - * @param props.searchResultsMetadataCollection - */ -const updateSearchResultsMeta = async ({ - fields, - jobId, - lastSignal, - logger, - searchResultsMetadataCollection, -}: UpdateSearchResultsMetaProps) => { - const filter: Record = { - _id: jobId, - }; - - if (lastSignal) { - filter.lastSignal = lastSignal; - } - - const modifier = { - $set: fields, - }; - - logger.debug("SearchResultsMetadataCollection modifier = ", modifier); - await searchResultsMetadataCollection.updateOne(filter, modifier); -}; - /** * Updates the search signal when the specified job finishes. * @@ -106,8 +72,12 @@ const updateSearchSignalWhenJobsFinish = async ({ return; } - await updateSearchResultsMeta({ - fields: { + const filter = { + _id: searchJobId.toString(), + lastSignal: SEARCH_SIGNAL.RESP_QUERYING, + }; + const modifier = { + $set: { lastSignal: SEARCH_SIGNAL.RESP_DONE, errorMsg: errorMsg, numTotalResults: Math.min( @@ -115,11 +85,8 @@ const updateSearchSignalWhenJobsFinish = async ({ SEARCH_MAX_NUM_RESULTS ), }, - jobId: searchJobId.toString(), - lastSignal: SEARCH_SIGNAL.RESP_QUERYING, - logger: logger, - searchResultsMetadataCollection: searchResultsMetadataCollection, - }); + }; + await searchResultsMetadataCollection.updateOne(filter, modifier); }; /** @@ -161,6 +128,5 @@ const createMongoIndexes = async ({ export { createMongoIndexes, hasCollection, - updateSearchResultsMeta, updateSearchSignalWhenJobsFinish, }; From 5a26a33cbbc84194ccfcaec68842b1e7e018ff18 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 8 Aug 2025 18:05:55 +0000 Subject: [PATCH 14/15] latest --- components/webui/server/src/routes/api/search/typings.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/components/webui/server/src/routes/api/search/typings.ts b/components/webui/server/src/routes/api/search/typings.ts index 55fd671ecc..41a2a87e1f 100644 --- a/components/webui/server/src/routes/api/search/typings.ts +++ b/components/webui/server/src/routes/api/search/typings.ts @@ -18,14 +18,6 @@ import { */ const SEARCH_MAX_NUM_RESULTS = 1000; -type UpdateSearchResultsMetaProps = { - fields: Partial; - jobId: string; - lastSignal?: SEARCH_SIGNAL; - logger: FastifyBaseLogger; - searchResultsMetadataCollection: Collection; -}; - type UpdateSearchSignalWhenJobsFinishProps = { aggregationJobId: number; logger: FastifyBaseLogger; @@ -46,6 +38,5 @@ export { CreateMongoIndexesProps, SEARCH_MAX_NUM_RESULTS, SearchResultsMetadataDocument, - UpdateSearchResultsMetaProps, UpdateSearchSignalWhenJobsFinishProps, }; From adc0a0d11b3e15398aacbe2ca4e023427a62b4f7 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 8 Aug 2025 18:09:24 +0000 Subject: [PATCH 15/15] latest --- .../webui/server/src/routes/api/presto-search/index.ts | 2 +- components/webui/server/src/routes/api/search/typings.ts | 5 +---- components/webui/server/src/routes/api/search/utils.ts | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/components/webui/server/src/routes/api/presto-search/index.ts b/components/webui/server/src/routes/api/presto-search/index.ts index dd19997005..71346089d9 100644 --- a/components/webui/server/src/routes/api/presto-search/index.ts +++ b/components/webui/server/src/routes/api/presto-search/index.ts @@ -104,7 +104,6 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { } }, query: queryString, - timeout: null, state: (_, queryId, stats) => { request.log.info({ searchJobId: queryId, @@ -136,6 +135,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => { success: () => { request.log.info("Presto search succeeded"); }, + timeout: null, }); }); } catch (error) { diff --git a/components/webui/server/src/routes/api/search/typings.ts b/components/webui/server/src/routes/api/search/typings.ts index 41a2a87e1f..f74f27611f 100644 --- a/components/webui/server/src/routes/api/search/typings.ts +++ b/components/webui/server/src/routes/api/search/typings.ts @@ -7,10 +7,7 @@ import type { Db, } from "mongodb"; -import { - SEARCH_SIGNAL, - type SearchResultsMetadataDocument, -} from "../../../../../common/index.js"; +import {type SearchResultsMetadataDocument} from "../../../../../common/index.js"; /** diff --git a/components/webui/server/src/routes/api/search/utils.ts b/components/webui/server/src/routes/api/search/utils.ts index 8df0dec1be..ff4c00af7d 100644 --- a/components/webui/server/src/routes/api/search/utils.ts +++ b/components/webui/server/src/routes/api/search/utils.ts @@ -86,6 +86,7 @@ const updateSearchSignalWhenJobsFinish = async ({ ), }, }; + await searchResultsMetadataCollection.updateOne(filter, modifier); };