Skip to content

Commit f5e981a

Browse files
Bill-hbrhbrBill-hbrhbr
authored
Merge branch 'main' into modernize-liblzma-install
2 parents ae41525 + 619392f commit f5e981a

File tree

11 files changed

+243
-111
lines changed

11 files changed

+243
-111
lines changed

‎components/clp-package-utils/clp_package_utils/scripts/archive_manager.py‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
import logging
33
import subprocess
44
import sys
5-
import typing
65
from pathlib import Path
6+
from typing import Final, List, Optional
77

88
from clp_py_utils.clp_config import (
99
CLP_DEFAULT_DATASET_NAME,
@@ -14,7 +14,7 @@
1414
from clp_package_utils.general import (
1515
CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH,
1616
CLPConfig,
17-
CLPDockerMounts,
17+
DockerMount,
1818
dump_container_config,
1919
generate_container_config,
2020
generate_container_name,
@@ -26,18 +26,18 @@
2626
)
2727

2828
# Command/Argument Constants
29-
FIND_COMMAND: typing.Final[str] = "find"
30-
DEL_COMMAND: typing.Final[str] = "del"
31-
DEL_BY_IDS_SUBCOMMAND: typing.Final[str] = "by-ids"
32-
DEL_BY_FILTER_SUBCOMMAND: typing.Final[str] = "by-filter"
33-
BEGIN_TS_ARG: typing.Final[str] = "--begin-ts"
34-
END_TS_ARG: typing.Final[str] = "--end-ts"
35-
DRY_RUN_ARG: typing.Final[str] = "--dry-run"
29+
FIND_COMMAND: Final[str] = "find"
30+
DEL_COMMAND: Final[str] = "del"
31+
DEL_BY_IDS_SUBCOMMAND: Final[str] = "by-ids"
32+
DEL_BY_FILTER_SUBCOMMAND: Final[str] = "by-filter"
33+
BEGIN_TS_ARG: Final[str] = "--begin-ts"
34+
END_TS_ARG: Final[str] = "--end-ts"
35+
DRY_RUN_ARG: Final[str] = "--dry-run"
3636

3737
logger: logging.Logger = logging.getLogger(__file__)
3838

3939

40-
def _validate_timestamps(begin_ts: int, end_ts: typing.Optional[int]) -> bool:
40+
def _validate_timestamps(begin_ts: int, end_ts: Optional[int]) -> bool:
4141
if begin_ts < 0:
4242
logger.error("begin-ts must be non-negative.")
4343
return False
@@ -50,7 +50,7 @@ def _validate_timestamps(begin_ts: int, end_ts: typing.Optional[int]) -> bool:
5050
return True
5151

5252

53-
def main(argv: typing.List[str]) -> int:
53+
def main(argv: List[str]) -> int:
5454
clp_home: Path = get_clp_home()
5555
default_config_file_path: Path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH
5656

@@ -159,8 +159,8 @@ def main(argv: typing.List[str]) -> int:
159159
else:
160160
logger.setLevel(logging.INFO)
161161

162-
begin_timestamp: typing.Optional[int]
163-
end_timestamp: typing.Optional[int]
162+
begin_timestamp: Optional[int] = None
163+
end_timestamp: Optional[int] = None
164164
subcommand: str = parsed_args.subcommand
165165

166166
# Validate and load config file
@@ -215,17 +215,17 @@ def main(argv: typing.List[str]) -> int:
215215
container_clp_config, clp_config, container_name
216216
)
217217

218-
necessary_mounts: typing.List[CLPDockerMounts] = [
218+
necessary_mounts: List[Optional[DockerMount]] = [
219219
mounts.clp_home,
220220
mounts.logs_dir,
221221
mounts.archives_output_dir,
222222
]
223-
container_start_cmd: typing.List[str] = generate_container_start_cmd(
223+
container_start_cmd: List[str] = generate_container_start_cmd(
224224
container_name, necessary_mounts, clp_config.execution_container
225225
)
226226

227227
# fmt: off
228-
archive_manager_cmd: typing.List[str] = [
228+
archive_manager_cmd: List[str] = [
229229
"python3",
230230
"-m", "clp_package_utils.scripts.native.archive_manager",
231231
"--config", str(generated_config_path_on_container),
@@ -264,7 +264,7 @@ def main(argv: typing.List[str]) -> int:
264264
logger.error(f"Unsupported subcommand: `{subcommand}`.")
265265
return -1
266266

267-
cmd: typing.List[str] = container_start_cmd + archive_manager_cmd
267+
cmd: List[str] = container_start_cmd + archive_manager_cmd
268268

269269
proc = subprocess.run(cmd)
270270
ret_code = proc.returncode

‎components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
import logging
33
import shutil
44
import sys
5-
import typing
65
from abc import ABC, abstractmethod
76
from contextlib import closing
87
from pathlib import Path
8+
from typing import Any, List, Optional
99

1010
from clp_py_utils.clp_config import Database
1111
from clp_py_utils.clp_metadata_db_utils import (
@@ -35,10 +35,10 @@
3535

3636

3737
class DeleteHandler(ABC):
38-
def __init__(self, query_params: typing.List[str]):
39-
self._params: typing.List[str] = query_params
38+
def __init__(self, query_params: List[str]):
39+
self._params: List[str] = query_params
4040

41-
def get_params(self) -> typing.List[str]:
41+
def get_params(self) -> List[str]:
4242
return self._params
4343

4444
@abstractmethod
@@ -48,7 +48,7 @@ def get_criteria(self) -> str: ...
4848
def get_not_found_message(self) -> str: ...
4949

5050
@abstractmethod
51-
def validate_results(self, archive_ids: typing.List[str]) -> None: ...
51+
def validate_results(self, archive_ids: List[str]) -> None: ...
5252

5353

5454
class FilterDeleteHandler(DeleteHandler):
@@ -58,7 +58,7 @@ def get_criteria(self) -> str:
5858
def get_not_found_message(self) -> str:
5959
return "No archives found within the specified time range."
6060

61-
def validate_results(self, archive_ids: typing.List[str]) -> None:
61+
def validate_results(self, archive_ids: List[str]) -> None:
6262
pass
6363

6464

@@ -70,7 +70,7 @@ def get_criteria(self) -> str:
7070
def get_not_found_message(self) -> str:
7171
return "No archives found with matching IDs."
7272

73-
def validate_results(self, archive_ids: typing.List[str]) -> None:
73+
def validate_results(self, archive_ids: List[str]) -> None:
7474
not_found_ids: set[str] = set(self._params) - set(archive_ids)
7575
if not_found_ids:
7676
logger.warning(
@@ -81,7 +81,7 @@ def validate_results(self, archive_ids: typing.List[str]) -> None:
8181
)
8282

8383

84-
def main(argv: typing.List[str]) -> int:
84+
def main(argv: List[str]) -> int:
8585
clp_home: Path = get_clp_home()
8686
default_config_file_path: Path = clp_home / CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH
8787

@@ -253,9 +253,9 @@ def main(argv: typing.List[str]) -> int:
253253
def _find_archives(
254254
archives_dir: Path,
255255
database_config: Database,
256-
dataset: typing.Optional[str],
256+
dataset: Optional[str],
257257
begin_ts: int,
258-
end_ts: int = typing.Optional[int],
258+
end_ts: int,
259259
) -> int:
260260
"""
261261
Lists all archive IDs, if begin_ts and end_ts are provided, only lists archives where
@@ -267,20 +267,20 @@ def _find_archives(
267267
:param end_ts:
268268
:return: 0 on success, 1 on failure.
269269
"""
270-
archive_ids: typing.List[str]
270+
archive_ids: List[str]
271271
dataset_specific_message = f" of dataset `{dataset}`" if dataset is not None else ""
272272
logger.info(f"Starting to find archives{dataset_specific_message} from the database.")
273273
try:
274274
sql_adapter: SQL_Adapter = SQL_Adapter(database_config)
275-
clp_db_connection_params: dict[str, any] = (
275+
clp_db_connection_params: dict[str, Any] = (
276276
database_config.get_clp_connection_params_and_type(True)
277277
)
278278
table_prefix: str = clp_db_connection_params["table_prefix"]
279279

280280
with closing(sql_adapter.create_connection(True)) as db_conn, closing(
281281
db_conn.cursor(dictionary=True)
282282
) as db_cursor:
283-
query_params: typing.List[int] = [begin_ts]
283+
query_params: List[int] = [begin_ts]
284284
query: str = (
285285
f"""
286286
SELECT id FROM `{get_archives_table_name(table_prefix, dataset)}`
@@ -294,7 +294,7 @@ def _find_archives(
294294
db_cursor.execute(query, query_params)
295295
results = db_cursor.fetchall()
296296

297-
archive_ids: typing.List[str] = [result["id"] for result in results]
297+
archive_ids: List[str] = [result["id"] for result in results]
298298
if 0 == len(archive_ids):
299299
logger.info("No archives found within specified time range.")
300300
return 0
@@ -318,7 +318,7 @@ def _find_archives(
318318
def _delete_archives(
319319
archives_dir: Path,
320320
database_config: Database,
321-
dataset: typing.Optional[str],
321+
dataset: Optional[str],
322322
delete_handler: DeleteHandler,
323323
dry_run: bool = False,
324324
) -> int:
@@ -333,11 +333,11 @@ def _delete_archives(
333333
:return: 0 on success, -1 otherwise.
334334
"""
335335

336-
archive_ids: typing.List[str]
336+
archive_ids: List[str]
337337
dataset_specific_message = f" of dataset `{dataset}`" if dataset is not None else ""
338338
logger.info(f"Starting to delete archives{dataset_specific_message} from the database.")
339339
sql_adapter: SQL_Adapter = SQL_Adapter(database_config)
340-
clp_db_connection_params: dict[str, any] = database_config.get_clp_connection_params_and_type(
340+
clp_db_connection_params: dict[str, Any] = database_config.get_clp_connection_params_and_type(
341341
True
342342
)
343343
table_prefix = clp_db_connection_params["table_prefix"]
@@ -350,7 +350,7 @@ def _delete_archives(
350350
logger.info("Running in dry-run mode.")
351351

352352
query_criteria: str = delete_handler.get_criteria()
353-
query_params: typing.List[str] = delete_handler.get_params()
353+
query_params: List[str] = delete_handler.get_params()
354354

355355
db_cursor.execute(
356356
f"""
@@ -365,7 +365,7 @@ def _delete_archives(
365365
logger.info(delete_handler.get_not_found_message())
366366
return 0
367367

368-
archive_ids: typing.List[str] = [result["id"] for result in results]
368+
archive_ids: List[str] = [result["id"] for result in results]
369369
delete_handler.validate_results(archive_ids)
370370

371371
delete_archives_from_metadata_db(db_cursor, archive_ids, table_prefix, dataset)

‎components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.module.css‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
.prestoProgress {
2+
position: absolute;
3+
line-height: 0;
4+
}

‎components/webui/client/src/pages/SearchPage/Presto/ProgressBar/index.tsx‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import {useEffect} from "react";
2+
3+
import {Progress} from "antd";
4+
5+
import useSearchStore from "../../SearchState";
6+
import {SEARCH_UI_STATE} from "../../SearchState/typings";
7+
import {usePseudoProgress} from "../../SearchState/usePseudoProgress";
8+
import styles from "./index.module.css";
9+
10+
11+
/**
12+
* Renders a pseudo progress bar that listens for changes in `searchUiState`.
13+
*
14+
* @return
15+
*/
16+
const ProgressBar = () => {
17+
const {searchUiState} = useSearchStore.getState();
18+
const {
19+
progress,
20+
start,
21+
stop,
22+
} = usePseudoProgress();
23+
24+
useEffect(() => {
25+
if (searchUiState === SEARCH_UI_STATE.QUERY_ID_PENDING) {
26+
start();
27+
} else if (
28+
searchUiState === SEARCH_UI_STATE.DONE ||
29+
searchUiState === SEARCH_UI_STATE.FAILED
30+
) {
31+
stop();
32+
}
33+
}, [searchUiState,
34+
start,
35+
stop]);
36+
37+
return (
38+
<Progress
39+
className={styles["prestoProgress"] || ""}
40+
percent={progress ?? 0}
41+
showInfo={false}
42+
size={"small"}
43+
status={"active"}
44+
strokeLinecap={"butt"}
45+
style={{display: null === progress ?
46+
"none" :
47+
"block"}}/>
48+
);
49+
};
50+
51+
export {ProgressBar};

‎components/webui/client/src/pages/SearchPage/SearchControls/QueryInput/index.tsx‎

Lines changed: 7 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,14 @@ import {
33
useCallback,
44
useEffect,
55
useRef,
6-
useState,
76
} from "react";
87

98
import type {InputRef} from "antd";
10-
import {Nullable} from "src/typings/common";
119

1210
import QueryBox from "../../../../components/QueryBox";
1311
import useSearchStore from "../../SearchState/index";
1412
import {SEARCH_UI_STATE} from "../../SearchState/typings";
15-
import {
16-
PROGRESS_INCREMENT,
17-
PROGRESS_INTERVAL_MILLIS,
18-
} from "./typings";
13+
import {usePseudoProgress} from "../../SearchState/usePseudoProgress";
1914

2015

2116
/**
@@ -27,8 +22,7 @@ const QueryInput = () => {
2722
const queryIsCaseSensitive = useSearchStore((state) => state.queryIsCaseSensitive);
2823
const queryString = useSearchStore((state) => state.queryString);
2924
const searchUiState = useSearchStore((state) => state.searchUiState);
30-
const [pseudoProgress, setPseudoProgress] = useState<Nullable<number>>(null);
31-
const intervalIdRef = useRef<number>(0);
25+
const {progress: pseudoProgress, start, stop} = usePseudoProgress();
3226
const inputRef = useRef<InputRef>(null);
3327

3428
const handleCaseSensitiveChange = useCallback((newValue: boolean) => {
@@ -43,34 +37,16 @@ const QueryInput = () => {
4337

4438
useEffect(() => {
4539
if (searchUiState === SEARCH_UI_STATE.QUERY_ID_PENDING) {
46-
if (0 !== intervalIdRef.current) {
47-
console.warn("Interval already set for submitted query");
48-
49-
return;
50-
}
51-
intervalIdRef.current = window.setInterval(() => {
52-
setPseudoProgress((v) => {
53-
if (100 <= (v ?? 0) + PROGRESS_INCREMENT) {
54-
return 100;
55-
}
56-
57-
return (v ?? 0) + PROGRESS_INCREMENT;
58-
});
59-
}, PROGRESS_INTERVAL_MILLIS);
40+
start();
6041
} else if (
6142
searchUiState === SEARCH_UI_STATE.DONE ||
6243
searchUiState === SEARCH_UI_STATE.FAILED
6344
) {
64-
clearInterval(intervalIdRef.current);
65-
intervalIdRef.current = 0;
66-
setPseudoProgress(null);
45+
stop();
6746
}
68-
}, [searchUiState]);
69-
70-
// Clear the interval if the component unmounts.
71-
useEffect(() => {
72-
clearInterval(intervalIdRef.current);
73-
}, []);
47+
}, [searchUiState,
48+
start,
49+
stop]);
7450

7551
useEffect(() => {
7652
if (

‎components/webui/client/src/pages/SearchPage/SearchControls/QueryInput/typings.ts‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)