diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index e98560891c..b988651fca 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -871,15 +871,20 @@ def start_webui( # Read, update, and write back client's and server's settings.json clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True) table_prefix = clp_db_connection_params["table_prefix"] - dataset: Optional[str] = None if StorageEngine.CLP_S == clp_config.package.storage_engine: - dataset = CLP_DEFAULT_DATASET_NAME + archives_table_name = "" + files_table_name = "" + else: + archives_table_name = get_archives_table_name(table_prefix, None) + files_table_name = get_files_table_name(table_prefix, None) + client_settings_json_updates = { "ClpStorageEngine": clp_config.package.storage_engine, "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name, - "SqlDbClpArchivesTableName": get_archives_table_name(table_prefix, dataset), + "SqlDbClpArchivesTableName": archives_table_name, "SqlDbClpDatasetsTableName": get_datasets_table_name(table_prefix), - "SqlDbClpFilesTableName": get_files_table_name(table_prefix, dataset), + "SqlDbClpFilesTableName": files_table_name, + "SqlDbClpTablePrefix": table_prefix, "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME, } client_settings_json = read_and_update_settings_json( diff --git a/components/webui/client/public/settings.json b/components/webui/client/public/settings.json index c55c147dbf..d2f7580d39 100644 --- a/components/webui/client/public/settings.json +++ b/components/webui/client/public/settings.json @@ -4,5 +4,6 @@ "SqlDbClpArchivesTableName": "clp_archives", "SqlDbClpDatasetsTableName": "clp_datasets", "SqlDbClpFilesTableName": "clp_files", + "SqlDbClpTablePrefix": "clp_", "SqlDbCompressionJobsTableName": "compression_jobs" } diff --git a/components/webui/client/src/App.tsx b/components/webui/client/src/App.tsx index c7d16c4f57..67d897c942 100644 --- a/components/webui/client/src/App.tsx +++ b/components/webui/client/src/App.tsx @@ -1,20 +1,16 @@ import {RouterProvider} from "react-router"; -import { - QueryClient, - QueryClientProvider, -} from "@tanstack/react-query"; +import {QueryClientProvider} from "@tanstack/react-query"; import {ReactQueryDevtools} from "@tanstack/react-query-devtools"; import {ConfigProvider} from "antd"; +import queryClient from "./config/queryClient"; import router from "./router"; import THEME_CONFIG from "./theme"; import "@ant-design/v5-patch-for-react-19"; -const queryClient = new QueryClient(); - /** * Renders Web UI app. * diff --git a/components/webui/client/src/api/query.ts b/components/webui/client/src/api/query/index.ts similarity index 97% rename from components/webui/client/src/api/query.ts rename to components/webui/client/src/api/query/index.ts index a00c6b57a6..eed0ff603a 100644 --- a/components/webui/client/src/api/query.ts +++ b/components/webui/client/src/api/query/index.ts @@ -7,7 +7,7 @@ import {Nullable} from "src/typings/common"; import { ExtractStreamResp, QUERY_JOB_TYPE, -} from "../typings/query"; +} from "../../typings/query"; interface SubmitExtractStreamJobProps { diff --git a/components/webui/client/src/api/search.ts b/components/webui/client/src/api/search/index.ts similarity index 97% rename from components/webui/client/src/api/search.ts rename to components/webui/client/src/api/search/index.ts index 8ebd176cb5..3f6e42973c 100644 --- a/components/webui/client/src/api/search.ts +++ b/components/webui/client/src/api/search/index.ts @@ -1,6 +1,6 @@ import axios, {AxiosResponse} from "axios"; -import {Nullable} from "../typings/common"; +import {Nullable} from "../../typings/common"; // eslint-disable-next-line no-warning-comments diff --git a/components/webui/client/src/api/sql.ts b/components/webui/client/src/api/sql/index.ts similarity index 100% rename from components/webui/client/src/api/sql.ts rename to components/webui/client/src/api/sql/index.ts diff --git a/components/webui/client/src/components/DashboardCard/index.tsx b/components/webui/client/src/components/DashboardCard/index.tsx index 692297f4b1..dc4a6bd840 100644 --- a/components/webui/client/src/components/DashboardCard/index.tsx +++ b/components/webui/client/src/components/DashboardCard/index.tsx @@ -13,6 +13,7 @@ interface DashboardCardProps { titleColor?: string; backgroundColor?: string; children?: React.ReactNode; + isLoading?: boolean; } /** @@ -23,13 +24,21 @@ interface DashboardCardProps { * @param props.titleColor * @param props.backgroundColor * @param props.children + * @param props.isLoading * @return */ -const DashboardCard = ({title, titleColor, backgroundColor, children}: DashboardCardProps) => { +const DashboardCard = ({ + title, + titleColor, + backgroundColor, + children, + isLoading = false, +}: DashboardCardProps) => { return (
diff --git a/components/webui/client/src/components/StatCard/index.tsx b/components/webui/client/src/components/StatCard/index.tsx index 4610c43946..8b5cff4c85 100644 --- a/components/webui/client/src/components/StatCard/index.tsx +++ b/components/webui/client/src/components/StatCard/index.tsx @@ -15,6 +15,7 @@ interface StatCardProps { backgroundColor?: string; statSize?: string; statColor?: string; + isLoading?: boolean; } /** @@ -27,6 +28,7 @@ interface StatCardProps { * @param props.backgroundColor * @param props.statSize * @param props.statColor + * @param props.isLoading * @return */ const StatCard = ({ @@ -36,9 +38,11 @@ const StatCard = ({ backgroundColor, statSize, statColor, + isLoading = false, }: StatCardProps) => { const props: DashboardCardProps = { title, + isLoading, ...(titleColor ? {titleColor} : {}), diff --git a/components/webui/client/src/config/queryClient.ts b/components/webui/client/src/config/queryClient.ts new file mode 100644 index 0000000000..1ce358f3c3 --- /dev/null +++ b/components/webui/client/src/config/queryClient.ts @@ -0,0 +1,14 @@ +import {QueryClient} from "@tanstack/react-query"; + + +const DEFAULT_REFETCH_INTERVAL_MILLIS = 10_000; + +const queryClient = new QueryClient({ + defaultOptions: { + queries: { + refetchInterval: DEFAULT_REFETCH_INTERVAL_MILLIS, + }, + }, +}); + +export default queryClient; diff --git a/components/webui/client/src/config/sql-table-suffix.ts b/components/webui/client/src/config/sql-table-suffix.ts new file mode 100644 index 0000000000..2449829bdf --- /dev/null +++ b/components/webui/client/src/config/sql-table-suffix.ts @@ -0,0 +1,7 @@ +/** + * Table suffixes for CLP database tables + */ +export enum SqlTableSuffix { + ARCHIVES = "archives", + FILES = "files", +} diff --git a/components/webui/client/src/pages/IngestPage/Details/DetailsCard.tsx b/components/webui/client/src/pages/IngestPage/Details/DetailsCard.tsx index b2e24e6497..fb497d0260 100644 --- a/components/webui/client/src/pages/IngestPage/Details/DetailsCard.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/DetailsCard.tsx @@ -6,6 +6,7 @@ import StatCard from "../../../components/StatCard"; interface DetailsCardProps { title: string; stat: string; + isLoading: boolean; } /** @@ -14,12 +15,14 @@ interface DetailsCardProps { * @param props * @param props.title * @param props.stat + * @param props.isLoading * @return */ -const DetailsCard = ({title, stat}: DetailsCardProps) => { +const DetailsCard = ({title, stat, isLoading}: DetailsCardProps) => { const {token} = theme.useToken(); return ( ; + isLoading: boolean; } /** @@ -12,11 +13,13 @@ interface FilesProps { * * @param props * @param props.numFiles + * @param props.isLoading * @return */ -const Files = ({numFiles}: FilesProps) => { +const Files = ({numFiles, isLoading}: FilesProps) => { return ( ); diff --git a/components/webui/client/src/pages/IngestPage/Details/Messages.tsx b/components/webui/client/src/pages/IngestPage/Details/Messages.tsx index 7eac3a23e5..8c7221b0ad 100644 --- a/components/webui/client/src/pages/IngestPage/Details/Messages.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/Messages.tsx @@ -5,6 +5,7 @@ import DetailsCard from "./DetailsCard"; interface MessagesProps { numMessages: Nullable; + isLoading: boolean; } /** @@ -12,11 +13,13 @@ interface MessagesProps { * * @param props * @param props.numMessages + * @param props.isLoading * @return */ -const Messages = ({numMessages}: MessagesProps) => { +const Messages = ({numMessages, isLoading}: MessagesProps) => { return ( ); diff --git a/components/webui/client/src/pages/IngestPage/Details/TimeRange.tsx b/components/webui/client/src/pages/IngestPage/Details/TimeRange.tsx index 8b05fd2f9b..1672cb4939 100644 --- a/components/webui/client/src/pages/IngestPage/Details/TimeRange.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/TimeRange.tsx @@ -8,6 +8,7 @@ const DATE_FORMAT = "MMMM D, YYYY"; interface TimeRangeProps { beginDate: Dayjs; endDate: Dayjs; + isLoading: boolean; } /** @@ -16,9 +17,10 @@ interface TimeRangeProps { * @param props * @param props.beginDate * @param props.endDate + * @param props.isLoading * @return */ -const TimeRange = ({beginDate, endDate}: TimeRangeProps) => { +const TimeRange = ({beginDate, endDate, isLoading}: TimeRangeProps) => { let stat; if (beginDate.isValid() && endDate.isValid()) { stat = `${beginDate.format(DATE_FORMAT)} - ${endDate.format(DATE_FORMAT)}`; @@ -28,6 +30,7 @@ const TimeRange = ({beginDate, endDate}: TimeRangeProps) => { return ( ); diff --git a/components/webui/client/src/pages/IngestPage/Details/index.tsx b/components/webui/client/src/pages/IngestPage/Details/index.tsx index 0f5eb7a0ce..cc94297f26 100644 --- a/components/webui/client/src/pages/IngestPage/Details/index.tsx +++ b/components/webui/client/src/pages/IngestPage/Details/index.tsx @@ -1,95 +1,64 @@ -import { - useCallback, - useEffect, - useState, -} from "react"; - -import dayjs, {Dayjs} from "dayjs"; -import {Nullable} from "src/typings/common"; +import {useQuery} from "@tanstack/react-query"; +import dayjs from "dayjs"; -import {querySql} from "../../../api/sql"; import { CLP_STORAGE_ENGINES, SETTINGS_STORAGE_ENGINE, } from "../../../config"; -import useIngestStatsStore from "../ingestStatsStore"; +import {fetchDatasetNames} from "../../SearchPage/SearchControls/Dataset/sql"; import Files from "./Files"; import styles from "./index.module.css"; import Messages from "./Messages"; import { - DetailsResp, - getDetailsSql, + DETAILS_DEFAULT, + fetchClpDetails, + fetchClpsDetails, } from "./sql"; import TimeRange from "./TimeRange"; -/** - * Default state for details. - */ -const DETAILS_DEFAULT = Object.freeze({ - beginDate: dayjs(null), - endDate: dayjs(null), - - numFiles: 0, - numMessages: 0, -}); - /** * Renders grid with compression details. * * @return */ const Details = () => { - const {refreshInterval} = useIngestStatsStore(); - const [beginDate, setBeginDate] = useState(DETAILS_DEFAULT.beginDate); - const [endDate, setEndDate] = useState(DETAILS_DEFAULT.endDate); - const [numFiles, setNumFiles] = useState>(DETAILS_DEFAULT.numFiles); - const [numMessages, setNumMessages] = useState>(DETAILS_DEFAULT.numMessages); + const {data: datasetNames = [], isSuccess: isSuccessDatasetNames} = useQuery({ + queryKey: ["datasets"], + queryFn: fetchDatasetNames, + enabled: CLP_STORAGE_ENGINES.CLP_S === SETTINGS_STORAGE_ENGINE, + }); - /** - * Fetches details stats from the server. - * - * @throws {Error} If the response is undefined. - */ - const fetchDetailsStats = useCallback(() => { - querySql(getDetailsSql()).then((resp) => { - const [details] = resp.data; - if ("undefined" === typeof details) { - throw new Error("Details response is undefined"); + const {data: details = DETAILS_DEFAULT, isPending} = useQuery({ + queryKey: [ + "details", + datasetNames, + ], + queryFn: async () => { + if (CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE) { + return fetchClpDetails(); } - setBeginDate(dayjs.utc(details.begin_timestamp)); - setEndDate(dayjs.utc(details.end_timestamp)); - setNumFiles(details.num_files); - setNumMessages(details.num_messages); - }) - .catch((e: unknown) => { - console.error("Failed to fetch details stats", e); - }); - }, []); - - - useEffect(() => { - fetchDetailsStats(); - const intervalId = setInterval(fetchDetailsStats, refreshInterval); - return () => { - clearInterval(intervalId); - }; - }, [ - refreshInterval, - fetchDetailsStats, - ]); + return fetchClpsDetails(datasetNames); + }, + enabled: CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE || isSuccessDatasetNames, + }); if (CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE) { return (
+ beginDate={dayjs.utc(details.begin_timestamp)} + endDate={dayjs.utc(details.end_timestamp)} + isLoading={isPending}/>
- - + +
); } @@ -97,8 +66,9 @@ const Details = () => { return (
+ beginDate={dayjs.utc(details.begin_timestamp)} + endDate={dayjs.utc(details.end_timestamp)} + isLoading={isPending}/>
); }; diff --git a/components/webui/client/src/pages/IngestPage/Details/sql.ts b/components/webui/client/src/pages/IngestPage/Details/sql.ts index 67970d0ce9..437fec822c 100644 --- a/components/webui/client/src/pages/IngestPage/Details/sql.ts +++ b/components/webui/client/src/pages/IngestPage/Details/sql.ts @@ -1,5 +1,7 @@ import {Nullable} from "src/typings/common"; +import {querySql} from "../../../api/sql"; +import {SqlTableSuffix} from "../../../config/sql-table-suffix"; import {settings} from "../../../settings"; import { CLP_ARCHIVES_TABLE_COLUMN_NAMES, @@ -8,7 +10,27 @@ import { /** - * Builds the query string to query stats. + * Result from SQL details query. + */ +interface DetailsItem { + begin_timestamp: Nullable; + end_timestamp: Nullable; + num_files: Nullable; + num_messages: Nullable; +} + +/** + * Default values for details when no data is available. + */ +const DETAILS_DEFAULT: DetailsItem = { + begin_timestamp: null, + end_timestamp: null, + num_files: 0, + num_messages: 0, +}; + +/** + * Builds the query string for details stats when using CLP storage engine (i.e. no datasets). * * @return */ @@ -38,17 +60,102 @@ FROM ) b; `; -interface DetailsItem { - begin_timestamp: Nullable; - end_timestamp: Nullable; - num_files: Nullable; - num_messages: Nullable; -} +/** + * Builds the query string for details stats when using CLP-S storage engine + * (i.e. multiple datasets). + * + * @param datasetNames + * @return + */ +const buildMultiDatasetDetailsSql = (datasetNames: string[]): string => { + const archiveQueries = datasetNames.map((name) => ` + SELECT + MIN(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.BEGIN_TIMESTAMP}) AS begin_timestamp, + MAX(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.END_TIMESTAMP}) AS end_timestamp + FROM ${settings.SqlDbClpTablePrefix}${name}_${SqlTableSuffix.ARCHIVES} + `); -type DetailsResp = DetailsItem[]; + const fileQueries = datasetNames.map((name) => ` + SELECT + COUNT(DISTINCT ${CLP_FILES_TABLE_COLUMN_NAMES.ORIG_FILE_ID}) AS num_files, + CAST( + COALESCE(SUM(${CLP_FILES_TABLE_COLUMN_NAMES.NUM_MESSAGES}), 0) AS INTEGER + ) AS num_messages + FROM ${settings.SqlDbClpTablePrefix}${name}_${SqlTableSuffix.FILES} + `); + + return ` + SELECT + a.begin_timestamp, + a.end_timestamp, + b.num_files, + b.num_messages + FROM + ( + SELECT + MIN(begin_timestamp) AS begin_timestamp, + MAX(end_timestamp) AS end_timestamp + FROM ( + ${archiveQueries.join("\nUNION ALL\n")} + ) AS archives_combined + ) a, + ( + SELECT + SUM(num_files) AS num_files, + SUM(num_messages) AS num_messages + FROM ( + ${fileQueries.join("\nUNION ALL\n")} + ) AS files_combined + ) b; + `; +}; + +/** + * Executes details SQL query and extracts details result. + * + * @param sql + * @return + * @throws {Error} if query result does not contain data + */ +const executeDetailsQuery = async (sql: string): Promise => { + const resp = await querySql(sql); + const [detailsResult] = resp.data; + if ("undefined" === typeof detailsResult) { + throw new Error("Details result does not contain data."); + } + + return detailsResult; +}; + +/** + * Fetches details statistics when using CLP storage engine. + * + * @return + */ +const fetchClpDetails = async (): Promise => { + const sql = getDetailsSql(); + return executeDetailsQuery(sql); +}; + +/** + * Fetches details statistics when using CLP-S storage engine. + * + * @param datasetNames + * @return + */ +const fetchClpsDetails = async ( + datasetNames: string[] +): Promise => { + if (0 === datasetNames.length) { + return DETAILS_DEFAULT; + } + const sql = buildMultiDatasetDetailsSql(datasetNames); + return executeDetailsQuery(sql); +}; -export type { - DetailsItem, - DetailsResp, +export type {DetailsItem}; +export { + DETAILS_DEFAULT, + fetchClpDetails, + fetchClpsDetails, }; -export {getDetailsSql}; diff --git a/components/webui/client/src/pages/IngestPage/Jobs/index.tsx b/components/webui/client/src/pages/IngestPage/Jobs/index.tsx index 48c5e08887..685aef96db 100644 --- a/components/webui/client/src/pages/IngestPage/Jobs/index.tsx +++ b/components/webui/client/src/pages/IngestPage/Jobs/index.tsx @@ -1,15 +1,9 @@ -import { - useCallback, - useEffect, - useState, -} from "react"; - +import {useQuery} from "@tanstack/react-query"; import dayjs from "dayjs"; import {querySql} from "../../../api/sql"; import {DashboardCard} from "../../../components/DashboardCard"; import VirtualTable from "../../../components/VirtualTable"; -import useIngestStatsStore from "../ingestStatsStore"; import styles from "./index.module.css"; import { getQueryJobsSql, @@ -24,13 +18,6 @@ import {convertQueryJobsItemToJobData} from "./utils"; const DAYS_TO_SHOW: number = 30; -/** - * Default state for jobs. - */ -const JOBS_DEFAULT = Object.freeze({ - jobs: [], -}); - interface JobsProps { className: string; } @@ -43,44 +30,15 @@ interface JobsProps { * @return */ const Jobs = ({className}: JobsProps) => { - const {refreshInterval} = useIngestStatsStore(); - const [jobs, setJobs] = useState(JOBS_DEFAULT.jobs); - - /** - * Fetches jobs stats from the server. - * - * @throws {Error} If the response is undefined. - */ - const fetchJobsStats = useCallback(() => { - const beginTimestamp = dayjs().subtract(DAYS_TO_SHOW, "days") - .unix(); - - querySql(getQueryJobsSql(beginTimestamp)) - .then((resp) => { - const newJobs = resp.data.map( - (item): JobData => convertQueryJobsItemToJobData(item) - ); - - setJobs(newJobs); - }) - .catch((e: unknown) => { - console.error("Failed to fetch jobs stats", e); - }); - }, []); - - - useEffect(() => { - fetchJobsStats(); - const intervalId = setInterval(fetchJobsStats, refreshInterval); - - return () => { - clearInterval(intervalId); - }; - }, [ - refreshInterval, - fetchJobsStats, - ]); - + const {data: jobs = [], isPending} = useQuery({ + queryKey: ["jobs"], + queryFn: async () => { + const beginTimestamp = dayjs().subtract(DAYS_TO_SHOW, "days") + .unix(); + const resp = await querySql(getQueryJobsSql(beginTimestamp)); + return resp.data.map((item): JobData => convertQueryJobsItemToJobData(item)); + }, + }); return (
@@ -89,6 +47,7 @@ const Jobs = ({className}: JobsProps) => { className={styles["jobs"] || ""} columns={jobColumns} dataSource={jobs} + loading={isPending} pagination={false} scroll={{y: 400}}/> diff --git a/components/webui/client/src/pages/IngestPage/SpaceSavings/CompressedSize.tsx b/components/webui/client/src/pages/IngestPage/SpaceSavings/CompressedSize.tsx index 8b9a9b3e38..3663ecc854 100644 --- a/components/webui/client/src/pages/IngestPage/SpaceSavings/CompressedSize.tsx +++ b/components/webui/client/src/pages/IngestPage/SpaceSavings/CompressedSize.tsx @@ -4,6 +4,7 @@ import {formatSizeInBytes} from "../Jobs/units"; interface CompressedSizeProps { compressedSize: number; + isLoading: boolean; } /** @@ -11,11 +12,13 @@ interface CompressedSizeProps { * * @param props * @param props.compressedSize + * @param props.isLoading * @return */ -const CompressedSize = ({compressedSize}: CompressedSizeProps) => { +const CompressedSize = ({compressedSize, isLoading}: CompressedSizeProps) => { return ( ); diff --git a/components/webui/client/src/pages/IngestPage/SpaceSavings/UncompressedSize.tsx b/components/webui/client/src/pages/IngestPage/SpaceSavings/UncompressedSize.tsx index 801d63a78f..c61471d724 100644 --- a/components/webui/client/src/pages/IngestPage/SpaceSavings/UncompressedSize.tsx +++ b/components/webui/client/src/pages/IngestPage/SpaceSavings/UncompressedSize.tsx @@ -4,6 +4,7 @@ import {formatSizeInBytes} from "../Jobs/units"; interface UncompressedSizeProps { uncompressedSize: number; + isLoading: boolean; } /** @@ -11,11 +12,13 @@ interface UncompressedSizeProps { * * @param props * @param props.uncompressedSize + * @param props.isLoading * @return */ -const UncompressedSize = ({uncompressedSize}: UncompressedSizeProps) => { +const UncompressedSize = ({uncompressedSize, isLoading}: UncompressedSizeProps) => { return ( ); diff --git a/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx b/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx index 112416686b..1d0d3e65f7 100644 --- a/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx +++ b/components/webui/client/src/pages/IngestPage/SpaceSavings/index.tsx @@ -1,72 +1,53 @@ -import { - useCallback, - useEffect, - useState, -} from "react"; - +import {useQuery} from "@tanstack/react-query"; import {theme} from "antd"; -import {querySql} from "../../../api/sql"; import StatCard from "../../../components/StatCard"; -import useIngestStatsStore from "../ingestStatsStore"; +import { + CLP_STORAGE_ENGINES, + SETTINGS_STORAGE_ENGINE, +} from "../../../config"; +import {fetchDatasetNames} from "../../SearchPage/SearchControls/Dataset/sql"; import CompressedSize from "./CompressedSize"; import styles from "./index.module.css"; import { - getSpaceSavingsSql, - SpaceSavingsResp, + fetchClpSpaceSavings, + fetchClpsSpaceSavings, + SPACE_SAVINGS_DEFAULT, } from "./sql"; import UncompressedSize from "./UncompressedSize"; -/** - * Default state for space savings. - */ -const SPACE_SAVINGS_DEFAULT = Object.freeze({ - compressedSize: 0, - uncompressedSize: 0, -}); - - /** * Renders space savings card. * * @return */ const SpaceSavings = () => { - const {refreshInterval} = useIngestStatsStore(); - const [compressedSize, setCompressedSize] = - useState(SPACE_SAVINGS_DEFAULT.compressedSize); - const [uncompressedSize, setUncompressedSize] = - useState(SPACE_SAVINGS_DEFAULT.uncompressedSize); const {token} = theme.useToken(); - const fetchSpaceSavingsStats = useCallback(() => { - querySql(getSpaceSavingsSql()) - .then((resp) => { - const [spaceSavings] = resp.data; - if ("undefined" === typeof spaceSavings) { - throw new Error("Space savings response is undefined"); - } - setCompressedSize(spaceSavings.total_compressed_size); - setUncompressedSize(spaceSavings.total_uncompressed_size); - }) - .catch((e: unknown) => { - console.error("Failed to fetch space savings stats", e); - }); - }, []); + const {data: datasetNames = [], isSuccess: isSuccessDatasetNames} = useQuery({ + queryKey: ["datasets"], + queryFn: fetchDatasetNames, + enabled: CLP_STORAGE_ENGINES.CLP_S === SETTINGS_STORAGE_ENGINE, + }); - useEffect(() => { - fetchSpaceSavingsStats(); - const intervalId = setInterval(fetchSpaceSavingsStats, refreshInterval); + const {data: spaceSavings = SPACE_SAVINGS_DEFAULT, isPending} = useQuery({ + queryKey: [ + "space-savings", + datasetNames, + ], + queryFn: async () => { + if (CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE) { + return fetchClpSpaceSavings(); + } - return () => { - clearInterval(intervalId); - }; - }, [ - refreshInterval, - fetchSpaceSavingsStats, - ]); + return fetchClpsSpaceSavings(datasetNames); + }, + enabled: CLP_STORAGE_ENGINES.CLP === SETTINGS_STORAGE_ENGINE || isSuccessDatasetNames, + }); + const compressedSize = spaceSavings.total_compressed_size; + const uncompressedSize = spaceSavings.total_uncompressed_size; const spaceSavingsPercent = (0 !== uncompressedSize) ? 100 * (1 - (compressedSize / uncompressedSize)) : @@ -79,14 +60,19 @@ const SpaceSavings = () => {
- - + +
); }; diff --git a/components/webui/client/src/pages/IngestPage/SpaceSavings/sql.ts b/components/webui/client/src/pages/IngestPage/SpaceSavings/sql.ts index e3639071b8..18dad3dd72 100644 --- a/components/webui/client/src/pages/IngestPage/SpaceSavings/sql.ts +++ b/components/webui/client/src/pages/IngestPage/SpaceSavings/sql.ts @@ -1,9 +1,28 @@ +import {querySql} from "../../../api/sql"; +import {SqlTableSuffix} from "../../../config/sql-table-suffix"; import {settings} from "../../../settings"; import {CLP_ARCHIVES_TABLE_COLUMN_NAMES} from "../sqlConfig"; /** - * Builds the query string to query stats. + * Result from sql space savings query. + */ +interface SpaceSavingsItem { + total_uncompressed_size: number; + total_compressed_size: number; +} + +/** + * Default values for space savings when no data is available. + */ +const SPACE_SAVINGS_DEFAULT: SpaceSavingsItem = { + total_compressed_size: 0, + total_uncompressed_size: 0, +}; + + +/** + * Builds the query string for space savings stats when using CLP storage engine (i.e. no datasets). * * @return */ @@ -24,15 +43,87 @@ SELECT FROM ${settings.SqlDbClpArchivesTableName} `; -interface SpaceSavingsItem { - total_uncompressed_size: number; - total_compressed_size: number; -} +/** + * Builds the query string for space savings stats when using CLP-S storage + * engine (i.e. multiple datasets). + * + * @param datasetNames + * @return + */ +const buildMultiDatasetSpaceSavingsSql = (datasetNames: string[]): string => { + const archiveQueries = datasetNames.map((name) => ` + SELECT + ${CLP_ARCHIVES_TABLE_COLUMN_NAMES.UNCOMPRESSED_SIZE}, + ${CLP_ARCHIVES_TABLE_COLUMN_NAMES.SIZE} + FROM ${settings.SqlDbClpTablePrefix}${name}_${SqlTableSuffix.ARCHIVES} + `); -type SpaceSavingsResp = SpaceSavingsItem[]; + return ` + SELECT + CAST( + COALESCE( + SUM(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.UNCOMPRESSED_SIZE}), + 0 + ) AS UNSIGNED + ) AS total_uncompressed_size, + CAST( + COALESCE( + SUM(${CLP_ARCHIVES_TABLE_COLUMN_NAMES.SIZE}), + 0 + ) AS UNSIGNED + ) AS total_compressed_size + FROM ( + ${archiveQueries.join("\nUNION ALL\n")} + ) AS archives_combined + `; +}; + +/** + * Executes space savings SQL query and extracts space savings result. + * + * @param sql + * @return + * @throws {Error} if query result does not contain data + */ +const executeSpaceSavingsQuery = async (sql: string): Promise => { + const resp = await querySql(sql); + const [spaceSavingsResult] = resp.data; + if ("undefined" === typeof spaceSavingsResult) { + throw new Error("Space savings result does not contain data."); + } + + return spaceSavingsResult; +}; + +/** + * Fetches space savings statistics when using CLP storage engine. + * + * @return + */ +const fetchClpSpaceSavings = async (): Promise => { + const sql = getSpaceSavingsSql(); + return executeSpaceSavingsQuery(sql); +}; + +/** + * Fetches space savings statistics when using CLP-S storage engine. + * + * @param datasetNames + * @return + */ +const fetchClpsSpaceSavings = async ( + datasetNames: string[] +): Promise => { + if (0 === datasetNames.length) { + return SPACE_SAVINGS_DEFAULT; + } + const sql = buildMultiDatasetSpaceSavingsSql(datasetNames); + return executeSpaceSavingsQuery(sql); +}; -export type { - SpaceSavingsItem, - SpaceSavingsResp, +export type {SpaceSavingsItem}; +export { + fetchClpSpaceSavings, + fetchClpsSpaceSavings, + SPACE_SAVINGS_DEFAULT, }; -export {getSpaceSavingsSql}; diff --git a/components/webui/client/src/pages/IngestPage/ingestStatsStore.ts b/components/webui/client/src/pages/IngestPage/ingestStatsStore.ts deleted file mode 100644 index 3aea09deb9..0000000000 --- a/components/webui/client/src/pages/IngestPage/ingestStatsStore.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {create} from "zustand"; - - -interface IngestStatsValues { - refreshInterval: number; -} - -interface IngestStatsActions { - setRefreshInterval: (newRefreshInterval: number)=> void; -} - -/** - * Default values for the ingest stats store. - */ -const INGEST_STATS_DEFAULT: IngestStatsValues = Object.freeze({ - refreshInterval: 10_000, -}); - -type IngestStatsState = IngestStatsValues & IngestStatsActions; - -const useIngestStatsStore = create((set) => ({ - ...INGEST_STATS_DEFAULT, - setRefreshInterval: (newRefreshInterval: number) => { - set(() => ({refreshInterval: newRefreshInterval})); - }, -})); - - -export default useIngestStatsStore; diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/index.tsx b/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/index.tsx index 2c00e71126..54c7e5a36c 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/index.tsx +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/index.tsx @@ -6,7 +6,6 @@ import { Select, } from "antd"; -import useIngestStatsStore from "../../../IngestPage/ingestStatsStore"; import useSearchStore from "../../SearchState/index"; import {SEARCH_UI_STATE} from "../../SearchState/typings"; import DatasetLabel from "./DatasetLabel"; @@ -20,8 +19,6 @@ import {fetchDatasetNames} from "./sql"; * @return */ const Dataset = () => { - const {refreshInterval} = useIngestStatsStore(); - const dataset = useSearchStore((state) => state.selectDataset); const updateDataset = useSearchStore((state) => state.updateSelectDataset); const searchUiState = useSearchStore((state) => state.searchUiState); @@ -31,7 +28,6 @@ const Dataset = () => { const {data, isPending, isSuccess, error} = useQuery({ queryKey: ["datasets"], queryFn: fetchDatasetNames, - staleTime: refreshInterval, }); // Update the selected dataset to the first dataset in the response. The dataset is only diff --git a/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/sql.ts b/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/sql.ts index 9bc7a622cc..8b0683105e 100644 --- a/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/sql.ts +++ b/components/webui/client/src/pages/SearchPage/SearchControls/Dataset/sql.ts @@ -10,7 +10,7 @@ enum CLP_DATASETS_TABLE_COLUMN_NAMES { } /** - * SQL query to get all datasets names. + * SQL query to get all dataset names. */ const GET_DATASETS_SQL = ` SELECT @@ -24,7 +24,7 @@ interface DatasetItem { } /** - * Fetches all datasets names from the datasets table. + * Fetches all dataset names from the datasets table. * * @return */ diff --git a/components/webui/client/src/settings.ts b/components/webui/client/src/settings.ts index dcc60208c1..53ac0949e8 100644 --- a/components/webui/client/src/settings.ts +++ b/components/webui/client/src/settings.ts @@ -7,6 +7,7 @@ type Settings = { SqlDbClpArchivesTableName: string; SqlDbClpDatasetsTableName: string; SqlDbClpFilesTableName: string; + SqlDbClpTablePrefix: string; SqlDbCompressionJobsTableName: string; };