From 2e9cdde4377d0dfd45f5ffaec51cdf7b0eedda53 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 9 May 2021 19:42:56 -0400 Subject: [PATCH 01/11] feat: add monitor using rich (plot table so far) --- setup.cfg | 1 + src/plotman/monitor.py | 136 +++++++++++++++++++++++++++++++++++++++++ src/plotman/plotman.py | 7 ++- 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 src/plotman/monitor.py diff --git a/setup.cfg b/setup.cfg index 98e6f84f..5aa68f73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ install_requires = pendulum psutil ~= 5.8 pyyaml + rich texttable [options.packages.find] diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py new file mode 100644 index 00000000..2f4984f7 --- /dev/null +++ b/src/plotman/monitor.py @@ -0,0 +1,136 @@ +import os +import time + +import attr +import rich +import rich.layout +import rich.live +import rich.table + +import plotman.configuration +import plotman.job +import plotman.plot_util +import plotman.reporting + + +def main(): + config_path = plotman.configuration.get_path() + config_text = plotman.configuration.read_configuration_text(config_path) + cfg = plotman.configuration.get_validated_configs(config_text, config_path) + + tmp_prefix = os.path.commonpath(cfg.directories.tmp) + dst_prefix = os.path.commonpath(cfg.directories.dst) + + overall = rich.layout.Layout('overall') + rows = [ + header_layout, + plots_layout, + disks_layout, + archive_layout, + logs_layout, + ] = [ + rich.layout.Layout(name='header'), + rich.layout.Layout(name='plots'), + rich.layout.Layout(name='disks'), + rich.layout.Layout(name='archive'), + rich.layout.Layout(name='logs'), + ] + overall.split_column(*rows) + + disks_layouts = [ + tmp_layout, + dst_layout, + ] = [ + rich.layout.Layout(name='tmp'), + rich.layout.Layout(name='dst'), + ] + + disks_layout.split_row(*disks_layouts) + + jobs = [] + + with rich.live.Live(overall, auto_refresh=False) as live: + for i in range(5): + tmp_layout.update(str(i)) + + jobs = plotman.job.Job.get_running_jobs( + cfg.directories.log, + cached_jobs=jobs, + ) + jobs_data = build_jobs_data( + jobs=jobs, + dst_prefix=dst_prefix, + tmp_prefix=tmp_prefix, + ) + + jobs_table = build_jobs_table(jobs_data=jobs_data) + plots_layout.update(jobs_table) + + live.refresh() + time.sleep(1) + + +def job_row_ib(name): + return attr.ib(converter=str, metadata={'name': name}) + + +@attr.frozen +class JobRow: + plot_id: str = job_row_ib(name='plot_id') + k: str = job_row_ib(name='k') + tmp_path: str = job_row_ib(name='tmp') + dst: str = job_row_ib(name='dst') + wall: str = job_row_ib(name='wall') + phase: str = job_row_ib(name='phase') + tmp_usage: str = job_row_ib(name='tmp') + pid: str = job_row_ib(name='pid') + stat: str = job_row_ib(name='stat') + mem: str = job_row_ib(name='mem') + user: str = job_row_ib(name='user') + sys: str = job_row_ib(name='sys') + io: str = job_row_ib(name='io') + + @classmethod + def from_job(cls, job, dst_prefix, tmp_prefix): + self = cls( + plot_id=job.plot_id[:8], + k=job.k, + tmp_path=plotman.reporting.abbr_path(job.tmpdir, tmp_prefix), + dst=plotman.reporting.abbr_path(job.dstdir, dst_prefix), + wall=plotman.plot_util.time_format(job.get_time_wall()), + phase=plotman.reporting.phase_str(job.progress()), + tmp_usage=plotman.plot_util.human_format(job.get_tmp_usage(), 0), + pid=job.proc.pid, + stat=job.get_run_status(), + mem=plotman.plot_util.human_format(job.get_mem_usage(), 1), + user=plotman.plot_util.time_format(job.get_time_user()), + sys=plotman.plot_util.time_format(job.get_time_sys()), + io=plotman.plot_util.time_format(job.get_time_iowait()) + ) + + return self + + +def build_jobs_data(jobs, dst_prefix, tmp_prefix): + sorted_jobs = sorted(jobs, key=plotman.job.Job.get_time_wall) + + jobs_data = [ + JobRow.from_job(job=job, dst_prefix=dst_prefix, tmp_prefix=tmp_prefix) + for index, job in enumerate(sorted_jobs) + ] + + return jobs_data + + +def build_jobs_table(jobs_data): + table = rich.table.Table(box=None, header_style='reverse') + + table.add_column('#') + + for field in attr.fields(JobRow): + table.add_column(field.metadata['name']) + + for index, row in enumerate(jobs_data): + table.add_row(str(index), *attr.astuple(row)) + + return table diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index d3468735..ccdb7e8b 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -7,7 +7,7 @@ import time # Plotman libraries -from plotman import analyzer, archive, configuration, interactive, manager, plot_util, reporting +from plotman import analyzer, archive, configuration, interactive, manager, monitor, plot_util, reporting from plotman import resources as plotman_resources from plotman.job import Job @@ -32,6 +32,8 @@ def parse_args(self): sp.add_parser('interactive', help='run interactive control/monitoring mode') + sp.add_parser('monitor', help='pure passive monitoring') + sp.add_parser('dsched', help='print destination dir schedule') sp.add_parser('plot', help='run plotting loop') @@ -158,6 +160,9 @@ def main(): analyzer.analyze(args.logfile, args.clipterminals, args.bytmp, args.bybitfield) + elif args.cmd == 'monitor': + monitor.main() + else: jobs = Job.get_running_jobs(cfg.directories.log) From 8cf6eac50f5f69fa19b8d019d0a8a46d01814815 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 9 May 2021 22:45:04 -0400 Subject: [PATCH 02/11] feat: q/ctrl+c quits monitor (but ctrl+c otherwise disabled) --- setup.cfg | 1 + src/plotman/monitor.py | 49 ++++++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5aa68f73..88f06190 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ install_requires = desert marshmallow pendulum + prompt_toolkit psutil ~= 5.8 pyyaml rich diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index 2f4984f7..a140413b 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -1,7 +1,11 @@ +import itertools import os import time import attr +import prompt_toolkit +import prompt_toolkit.input +import prompt_toolkit.keys import rich import rich.layout import rich.live @@ -49,25 +53,32 @@ def main(): jobs = [] - with rich.live.Live(overall, auto_refresh=False) as live: - for i in range(5): - tmp_layout.update(str(i)) - - jobs = plotman.job.Job.get_running_jobs( - cfg.directories.log, - cached_jobs=jobs, - ) - jobs_data = build_jobs_data( - jobs=jobs, - dst_prefix=dst_prefix, - tmp_prefix=tmp_prefix, - ) - - jobs_table = build_jobs_table(jobs_data=jobs_data) - plots_layout.update(jobs_table) - - live.refresh() - time.sleep(1) + prompt_toolkit_input = prompt_toolkit.input.create_input() + with prompt_toolkit_input.raw_mode(): + with rich.live.Live(overall, auto_refresh=False) as live: + for i in itertools.count(): + tmp_layout.update(str(i)) + + jobs = plotman.job.Job.get_running_jobs( + cfg.directories.log, + cached_jobs=jobs, + ) + jobs_data = build_jobs_data( + jobs=jobs, + dst_prefix=dst_prefix, + tmp_prefix=tmp_prefix, + ) + + jobs_table = build_jobs_table(jobs_data=jobs_data) + plots_layout.update(jobs_table) + + live.refresh() + for _ in range(10): + keys = prompt_toolkit_input.read_keys() + quit_keys = {'q', prompt_toolkit.keys.Keys.ControlC} + if any(key.key in quit_keys for key in keys): + return + time.sleep(0.1) def job_row_ib(name): From 637bf314e786bdce8fd15c0af05bbf053f326a29 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 10 May 2021 20:32:06 -0400 Subject: [PATCH 03/11] rich and prompt_toolkit --- setup.cfg | 1 + src/plotman/monitor.py | 98 +++++++++++++++++++++++++++++++++++++++++- src/plotman/plotman.py | 12 ++++-- 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 88f06190..c7697bdb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -36,6 +36,7 @@ package_dir= =src packages=find: install_requires = + anyio appdirs attrs click diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index a140413b..b615ea57 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -1,12 +1,20 @@ +import functools +import io import itertools import os import time +import anyio import attr import prompt_toolkit +import prompt_toolkit.buffer import prompt_toolkit.input +import prompt_toolkit.key_binding import prompt_toolkit.keys +import prompt_toolkit.layout.containers +import prompt_toolkit.layout.layout import rich +import rich.console import rich.layout import rich.live import rich.table @@ -17,7 +25,7 @@ import plotman.reporting -def main(): +def with_rich(): config_path = plotman.configuration.get_path() config_text = plotman.configuration.read_configuration_text(config_path) cfg = plotman.configuration.get_validated_configs(config_text, config_path) @@ -81,6 +89,94 @@ def main(): time.sleep(0.1) +async def with_prompt_toolkit(): + config_path = plotman.configuration.get_path() + config_text = plotman.configuration.read_configuration_text(config_path) + cfg = plotman.configuration.get_validated_configs(config_text, config_path) + + tmp_prefix = os.path.commonpath(cfg.directories.tmp) + dst_prefix = os.path.commonpath(cfg.directories.dst) + + plots_buffer = prompt_toolkit.layout.controls.Buffer() + disks_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + rows = [ + header_window, + plots_window, + disks_window, + archive_window, + logs_window, + ] = [ + prompt_toolkit.layout.containers.Window(), + prompt_toolkit.layout.containers.Window( + prompt_toolkit.layout.controls.BufferControl(buffer=plots_buffer), + ), + prompt_toolkit.layout.containers.Window(content=disks_buffer), + prompt_toolkit.layout.containers.Window(), + prompt_toolkit.layout.containers.Window(), + ] + + root_container = prompt_toolkit.layout.containers.HSplit(rows) + + layout = prompt_toolkit.layout.Layout(root_container) + + key_bindings = prompt_toolkit.key_binding.KeyBindings() + + key_bindings.add('q')(exit_key_binding) + + application = prompt_toolkit.Application( + layout=layout, + full_screen=True, + key_bindings=key_bindings, + ) + + rich_console = rich.console.Console( + # file=io.StringIO, + force_terminal=True, + # color_system="truecolor", + ) + + jobs = [] + + async with anyio.move_on_after(10): + async with anyio.create_task_group() as task_group: + task_group.start_soon(functools.partial( + cancel_after_application, + application=application, + cancel_scope=task_group.cancel_scope, + )) + + for i in itertools.count(): + disks_buffer.text = str(i) + + jobs = plotman.job.Job.get_running_jobs( + cfg.directories.log, + cached_jobs=jobs, + ) + jobs_data = build_jobs_data( + jobs=jobs, + dst_prefix=dst_prefix, + tmp_prefix=tmp_prefix, + ) + + jobs_table = build_jobs_table(jobs_data=jobs_data) + with rich_console.capture() as capture: + rich_console.print(jobs_table) + + plots_buffer.text = capture.get() + + application.invalidate() + await anyio.sleep(1) + + +async def cancel_after_application(application, cancel_scope): + await application.run_async() + cancel_scope.cancel() + + +def exit_key_binding(event): + event.app.exit() + + def job_row_ib(name): return attr.ib(converter=str, metadata={'name': name}) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index ccdb7e8b..6d487004 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -1,3 +1,4 @@ +import anyio import argparse import importlib import importlib.resources @@ -32,7 +33,9 @@ def parse_args(self): sp.add_parser('interactive', help='run interactive control/monitoring mode') - sp.add_parser('monitor', help='pure passive monitoring') + sp.add_parser('rich', help='monitoring with rich') + + sp.add_parser('prompt_toolkit', help='monitoring with prompt_toolkit') sp.add_parser('dsched', help='print destination dir schedule') @@ -160,8 +163,11 @@ def main(): analyzer.analyze(args.logfile, args.clipterminals, args.bytmp, args.bybitfield) - elif args.cmd == 'monitor': - monitor.main() + elif args.cmd == 'rich': + monitor.with_rich() + + elif args.cmd == 'prompt_toolkit': + anyio.run(monitor.with_prompt_toolkit) else: jobs = Job.get_running_jobs(cfg.directories.log) From 1d4fcb69b65cc1fa6dffcfe5029db6cbe3611e75 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 10 May 2021 22:26:30 -0400 Subject: [PATCH 04/11] fix prompt_toolkit rendering of rich output --- src/plotman/monitor.py | 69 +++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index b615ea57..6a81dd8a 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -97,7 +97,7 @@ async def with_prompt_toolkit(): tmp_prefix = os.path.commonpath(cfg.directories.tmp) dst_prefix = os.path.commonpath(cfg.directories.dst) - plots_buffer = prompt_toolkit.layout.controls.Buffer() + plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl() disks_buffer = prompt_toolkit.layout.controls.FormattedTextControl() rows = [ header_window, @@ -107,9 +107,7 @@ async def with_prompt_toolkit(): logs_window, ] = [ prompt_toolkit.layout.containers.Window(), - prompt_toolkit.layout.containers.Window( - prompt_toolkit.layout.controls.BufferControl(buffer=plots_buffer), - ), + prompt_toolkit.layout.containers.Window(content=plots_buffer), prompt_toolkit.layout.containers.Window(content=disks_buffer), prompt_toolkit.layout.containers.Window(), prompt_toolkit.layout.containers.Window(), @@ -129,43 +127,38 @@ async def with_prompt_toolkit(): key_bindings=key_bindings, ) - rich_console = rich.console.Console( - # file=io.StringIO, - force_terminal=True, - # color_system="truecolor", - ) + rich_console = rich.console.Console() jobs = [] - async with anyio.move_on_after(10): - async with anyio.create_task_group() as task_group: - task_group.start_soon(functools.partial( - cancel_after_application, - application=application, - cancel_scope=task_group.cancel_scope, - )) - - for i in itertools.count(): - disks_buffer.text = str(i) - - jobs = plotman.job.Job.get_running_jobs( - cfg.directories.log, - cached_jobs=jobs, - ) - jobs_data = build_jobs_data( - jobs=jobs, - dst_prefix=dst_prefix, - tmp_prefix=tmp_prefix, - ) - - jobs_table = build_jobs_table(jobs_data=jobs_data) - with rich_console.capture() as capture: - rich_console.print(jobs_table) - - plots_buffer.text = capture.get() - - application.invalidate() - await anyio.sleep(1) + async with anyio.create_task_group() as task_group: + task_group.start_soon(functools.partial( + cancel_after_application, + application=application, + cancel_scope=task_group.cancel_scope, + )) + + for i in itertools.count(): + disks_buffer.text = str(i) + + jobs = plotman.job.Job.get_running_jobs( + cfg.directories.log, + cached_jobs=jobs, + ) + jobs_data = build_jobs_data( + jobs=jobs, + dst_prefix=dst_prefix, + tmp_prefix=tmp_prefix, + ) + + jobs_table = build_jobs_table(jobs_data=jobs_data) + with rich_console.capture() as capture: + rich_console.print(jobs_table) + + plots_buffer.text = prompt_toolkit.ANSI(capture.get()) + + application.invalidate() + await anyio.sleep(1) async def cancel_after_application(application, cancel_scope): From a5846e34972b5c27dc99fd33c447e2dd2565bde1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 11 May 2021 20:05:11 -0400 Subject: [PATCH 05/11] q and ctrl+c --- src/plotman/monitor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index 6a81dd8a..d0db32ea 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -119,8 +119,6 @@ async def with_prompt_toolkit(): key_bindings = prompt_toolkit.key_binding.KeyBindings() - key_bindings.add('q')(exit_key_binding) - application = prompt_toolkit.Application( layout=layout, full_screen=True, @@ -132,6 +130,9 @@ async def with_prompt_toolkit(): jobs = [] async with anyio.create_task_group() as task_group: + key_bindings.add('q')(exit_key_binding) + key_bindings.add('c-c')(exit_key_binding) + task_group.start_soon(functools.partial( cancel_after_application, application=application, From d0cf089903d980017b9703b5df2a35c4086f909f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 12 May 2021 22:56:46 -0400 Subject: [PATCH 06/11] add tmp table --- src/plotman/monitor.py | 117 ++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 20 deletions(-) diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index d0db32ea..66f748ee 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -21,6 +21,7 @@ import plotman.configuration import plotman.job +import plotman.manager import plotman.plot_util import plotman.reporting @@ -65,7 +66,7 @@ def with_rich(): with prompt_toolkit_input.raw_mode(): with rich.live.Live(overall, auto_refresh=False) as live: for i in itertools.count(): - tmp_layout.update(str(i)) + header_layout.update(str(i)) jobs = plotman.job.Job.get_running_jobs( cfg.directories.log, @@ -80,6 +81,16 @@ def with_rich(): jobs_table = build_jobs_table(jobs_data=jobs_data) plots_layout.update(jobs_table) + tmp_data = build_tmp_data( + jobs=jobs, + dir_cfg=cfg.directories, + sched_cfg=cfg.scheduling, + prefix=tmp_prefix, + ) + + tmp_table = build_tmp_table(tmp_data=tmp_data) + tmp_layout.update(tmp_table) + live.refresh() for _ in range(10): keys = prompt_toolkit_input.read_keys() @@ -97,6 +108,7 @@ async def with_prompt_toolkit(): tmp_prefix = os.path.commonpath(cfg.directories.tmp) dst_prefix = os.path.commonpath(cfg.directories.dst) + header_buffer = prompt_toolkit.layout.controls.FormattedTextControl() plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl() disks_buffer = prompt_toolkit.layout.controls.FormattedTextControl() rows = [ @@ -106,7 +118,7 @@ async def with_prompt_toolkit(): archive_window, logs_window, ] = [ - prompt_toolkit.layout.containers.Window(), + prompt_toolkit.layout.containers.Window(content=header_buffer), prompt_toolkit.layout.containers.Window(content=plots_buffer), prompt_toolkit.layout.containers.Window(content=disks_buffer), prompt_toolkit.layout.containers.Window(), @@ -140,7 +152,7 @@ async def with_prompt_toolkit(): )) for i in itertools.count(): - disks_buffer.text = str(i) + header_buffer.text = str(i) jobs = plotman.job.Job.get_running_jobs( cfg.directories.log, @@ -153,10 +165,17 @@ async def with_prompt_toolkit(): ) jobs_table = build_jobs_table(jobs_data=jobs_data) - with rich_console.capture() as capture: - rich_console.print(jobs_table) + plots_buffer.text = capture_rich(jobs_table, console=rich_console) - plots_buffer.text = prompt_toolkit.ANSI(capture.get()) + tmp_data = build_tmp_data( + jobs=jobs, + dir_cfg=cfg.directories, + sched_cfg=cfg.scheduling, + prefix=tmp_prefix, + ) + + tmp_table = build_tmp_table(tmp_data=tmp_data) + disks_buffer.text = capture_rich(tmp_table, console=rich_console) application.invalidate() await anyio.sleep(1) @@ -171,25 +190,32 @@ def exit_key_binding(event): event.app.exit() -def job_row_ib(name): +def capture_rich(*objects, console): + with console.capture() as capture: + console.print(*objects) + + return prompt_toolkit.ANSI(capture.get()) + + +def row_ib(name): return attr.ib(converter=str, metadata={'name': name}) @attr.frozen class JobRow: - plot_id: str = job_row_ib(name='plot_id') - k: str = job_row_ib(name='k') - tmp_path: str = job_row_ib(name='tmp') - dst: str = job_row_ib(name='dst') - wall: str = job_row_ib(name='wall') - phase: str = job_row_ib(name='phase') - tmp_usage: str = job_row_ib(name='tmp') - pid: str = job_row_ib(name='pid') - stat: str = job_row_ib(name='stat') - mem: str = job_row_ib(name='mem') - user: str = job_row_ib(name='user') - sys: str = job_row_ib(name='sys') - io: str = job_row_ib(name='io') + plot_id: str = row_ib(name='plot id') + k: str = row_ib(name='k') + tmp_path: str = row_ib(name='tmp') + dst: str = row_ib(name='dst') + wall: str = row_ib(name='wall') + phase: str = row_ib(name='phase') + tmp_usage: str = row_ib(name='tmp') + pid: str = row_ib(name='pid') + stat: str = row_ib(name='stat') + mem: str = row_ib(name='mem') + user: str = row_ib(name='user') + sys: str = row_ib(name='sys') + io: str = row_ib(name='io') @classmethod def from_job(cls, job, dst_prefix, tmp_prefix): @@ -235,3 +261,54 @@ def build_jobs_table(jobs_data): table.add_row(str(index), *attr.astuple(row)) return table + + +@attr.frozen +class TmpRow: + path: str = row_ib(name='tmp') + ready: bool = row_ib(name='ready') + phases: list[plotman.job.Phase] = row_ib(name='phases') + + @classmethod + def from_tmp(cls, dir_cfg, jobs, sched_cfg, tmp, prefix): + phases = sorted(plotman.job.job_phases_for_tmpdir(d=tmp, all_jobs=jobs)) + tmp_suffix = plotman.reporting.abbr_path(path=tmp, putative_prefix=prefix) + ready = plotman.manager.phases_permit_new_job( + phases=phases, + d=tmp_suffix, + sched_cfg=sched_cfg, + dir_cfg=dir_cfg, + ) + self = cls( + path=tmp_suffix, + ready='OK' if ready else '--', + phases=plotman.reporting.phases_str(phases=phases, max_num=5), + ) + return self + + +def build_tmp_data(jobs, dir_cfg, sched_cfg, prefix): + rows = [ + TmpRow.from_tmp( + dir_cfg=dir_cfg, + jobs=jobs, + sched_cfg=sched_cfg, + tmp=tmp, + prefix=prefix, + ) + for tmp in sorted(dir_cfg.tmp) + ] + + return rows + + +def build_tmp_table(tmp_data): + table = rich.table.Table(box=None, header_style='reverse') + + for field in attr.fields(TmpRow): + table.add_column(field.metadata['name']) + + for row in tmp_data: + table.add_row(*attr.astuple(row)) + + return table From 14ddce2472d4c5ae0e0e8cd5c015b1228a05c4e4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 15 May 2021 22:52:01 -0400 Subject: [PATCH 07/11] add dst table --- src/plotman/monitor.py | 92 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 4 deletions(-) diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index 66f748ee..eaceedc8 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -19,6 +19,7 @@ import rich.live import rich.table +import plotman.archive import plotman.configuration import plotman.job import plotman.manager @@ -110,17 +111,25 @@ async def with_prompt_toolkit(): header_buffer = prompt_toolkit.layout.controls.FormattedTextControl() plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl() - disks_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + tmp_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + dst_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + disk_columns = [ + tmp_window, + dst_window, + ] = [ + prompt_toolkit.layout.containers.Window(content=tmp_buffer, dont_extend_width=True), + prompt_toolkit.layout.containers.Window(content=dst_buffer), + ] rows = [ header_window, plots_window, - disks_window, + disk_layout, archive_window, logs_window, ] = [ prompt_toolkit.layout.containers.Window(content=header_buffer), prompt_toolkit.layout.containers.Window(content=plots_buffer), - prompt_toolkit.layout.containers.Window(content=disks_buffer), + prompt_toolkit.layout.containers.VSplit(disk_columns, padding=1), prompt_toolkit.layout.containers.Window(), prompt_toolkit.layout.containers.Window(), ] @@ -137,6 +146,9 @@ async def with_prompt_toolkit(): key_bindings=key_bindings, ) + # https://github.com/prompt-toolkit/python-prompt-toolkit/issues/827#issuecomment-459099452 + application.output.show_cursor = lambda: False + rich_console = rich.console.Console() jobs = [] @@ -175,7 +187,16 @@ async def with_prompt_toolkit(): ) tmp_table = build_tmp_table(tmp_data=tmp_data) - disks_buffer.text = capture_rich(tmp_table, console=rich_console) + tmp_buffer.text = capture_rich(tmp_table, console=rich_console) + + dst_data = build_dst_data( + jobs=jobs, + dstdirs=cfg.directories.dst, + prefix=tmp_prefix, + ) + + dst_table = build_dst_table(dst_data=dst_data) + dst_buffer.text = capture_rich(dst_table, console=rich_console) application.invalidate() await anyio.sleep(1) @@ -312,3 +333,66 @@ def build_tmp_table(tmp_data): table.add_row(*attr.astuple(row)) return table + + +@attr.frozen +class DstRow: + dst: str = row_ib(name='dst') + plot_count: int = row_ib(name='plots') + free: int = row_ib(name='free') + inbound_phases: list[plotman.job.Phase] = row_ib(name='phases') + priority: int = row_ib(name='pri') + + @classmethod + def from_dst(cls, dst, jobs, prefix, dir2oldphase): + # TODO: This logic is replicated in archive.py's priority computation, + # maybe by moving more of the logic in to directory.py + eldest_ph = dir2oldphase.get(dst, plotman.job.Phase(0, 0)) + phases = plotman.job.job_phases_for_dstdir(dst, jobs) + + dir_plots = plotman.plot_util.list_k32_plots(dst) + free = plotman.plot_util.df_b(dst) + n_plots = len(dir_plots) + priority = plotman.archive.compute_priority( + phase=eldest_ph, + gb_free=free / plotman.plot_util.GB, + n_plots=n_plots, + ) + + self = cls( + dst=plotman.reporting.abbr_path(dst, prefix), + plot_count=n_plots, + free=plotman.plot_util.human_format(free, 0), + inbound_phases=plotman.reporting.phases_str(phases, 5), + priority=priority, + ) + + return self + + +def build_dst_data(jobs, dstdirs, prefix): + dir2oldphase = plotman.manager.dstdirs_to_furthest_phase(jobs) + + rows = [ + DstRow.from_dst( + dst=dst, + jobs=jobs, + prefix=prefix, + dir2oldphase=dir2oldphase, + ) + for dst in sorted(dstdirs) + ] + + return rows + + +def build_dst_table(dst_data): + table = rich.table.Table(box=None, header_style='reverse') + + for field in attr.fields(DstRow): + table.add_column(field.metadata['name']) + + for row in dst_data: + table.add_row(*attr.astuple(row)) + + return table From b40aa89825c3feb5fb2f59dda8a6c4d3c8d4c8eb Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 15 May 2021 23:18:55 -0400 Subject: [PATCH 08/11] add key bindings shortcuts list --- src/plotman/monitor.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index eaceedc8..5b5c01ac 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -1,3 +1,4 @@ +import collections import functools import io import itertools @@ -113,6 +114,9 @@ async def with_prompt_toolkit(): plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl() tmp_buffer = prompt_toolkit.layout.controls.FormattedTextControl() dst_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + archive_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + footer_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + disk_columns = [ tmp_window, dst_window, @@ -126,12 +130,14 @@ async def with_prompt_toolkit(): disk_layout, archive_window, logs_window, + footer_window, ] = [ prompt_toolkit.layout.containers.Window(content=header_buffer), prompt_toolkit.layout.containers.Window(content=plots_buffer), prompt_toolkit.layout.containers.VSplit(disk_columns, padding=1), + prompt_toolkit.layout.containers.Window(content=archive_buffer), prompt_toolkit.layout.containers.Window(), - prompt_toolkit.layout.containers.Window(), + prompt_toolkit.layout.containers.Window(content=footer_buffer, dont_extend_height=True), ] root_container = prompt_toolkit.layout.containers.HSplit(rows) @@ -153,10 +159,22 @@ async def with_prompt_toolkit(): jobs = [] - async with anyio.create_task_group() as task_group: - key_bindings.add('q')(exit_key_binding) - key_bindings.add('c-c')(exit_key_binding) + key_bindings.add('q', 'c-c')(exit_key_binding) + + binding_handler_names = { + exit_key_binding: 'exit', + } + key_bindings_text = ' '.join( + f'{binding_handler_names[binding.handler]} \\[{", ".join(binding.keys)}]' + for binding in key_bindings.bindings + ) + footer_buffer.text = capture_rich( + f'[reverse]key bindings[/reverse]\n{key_bindings_text}', + console=rich_console, + ) + + async with anyio.create_task_group() as task_group: task_group.start_soon(functools.partial( cancel_after_application, application=application, @@ -207,7 +225,7 @@ async def cancel_after_application(application, cancel_scope): cancel_scope.cancel() -def exit_key_binding(event): +async def exit_key_binding(event): event.app.exit() From ee557aa9f9e87c39a4e5cd8574effd80297229ad Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 15 May 2021 23:30:15 -0400 Subject: [PATCH 09/11] adjust window extension --- src/plotman/monitor.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index 5b5c01ac..d3a65194 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -110,19 +110,20 @@ async def with_prompt_toolkit(): tmp_prefix = os.path.commonpath(cfg.directories.tmp) dst_prefix = os.path.commonpath(cfg.directories.dst) - header_buffer = prompt_toolkit.layout.controls.FormattedTextControl() - plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl() - tmp_buffer = prompt_toolkit.layout.controls.FormattedTextControl() - dst_buffer = prompt_toolkit.layout.controls.FormattedTextControl() - archive_buffer = prompt_toolkit.layout.controls.FormattedTextControl() - footer_buffer = prompt_toolkit.layout.controls.FormattedTextControl() + header_buffer = prompt_toolkit.layout.controls.FormattedTextControl('header') + plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl('plots') + tmp_buffer = prompt_toolkit.layout.controls.FormattedTextControl('tmp') + dst_buffer = prompt_toolkit.layout.controls.FormattedTextControl('dst') + archive_buffer = prompt_toolkit.layout.controls.FormattedTextControl('archive') + logs_buffer = prompt_toolkit.layout.controls.FormattedTextControl('logs') + footer_buffer = prompt_toolkit.layout.controls.FormattedTextControl('footer') disk_columns = [ tmp_window, dst_window, ] = [ - prompt_toolkit.layout.containers.Window(content=tmp_buffer, dont_extend_width=True), - prompt_toolkit.layout.containers.Window(content=dst_buffer), + prompt_toolkit.layout.containers.Window(content=tmp_buffer, dont_extend_height=True, dont_extend_width=True), + prompt_toolkit.layout.containers.Window(content=dst_buffer, dont_extend_height=True), ] rows = [ header_window, @@ -132,11 +133,11 @@ async def with_prompt_toolkit(): logs_window, footer_window, ] = [ - prompt_toolkit.layout.containers.Window(content=header_buffer), - prompt_toolkit.layout.containers.Window(content=plots_buffer), + prompt_toolkit.layout.containers.Window(content=header_buffer, dont_extend_height=True), + prompt_toolkit.layout.containers.Window(content=plots_buffer, dont_extend_height=True), prompt_toolkit.layout.containers.VSplit(disk_columns, padding=1), - prompt_toolkit.layout.containers.Window(content=archive_buffer), - prompt_toolkit.layout.containers.Window(), + prompt_toolkit.layout.containers.Window(content=archive_buffer, dont_extend_height=True), + prompt_toolkit.layout.containers.Window(content=logs_buffer), prompt_toolkit.layout.containers.Window(content=footer_buffer, dont_extend_height=True), ] @@ -233,7 +234,7 @@ def capture_rich(*objects, console): with console.capture() as capture: console.print(*objects) - return prompt_toolkit.ANSI(capture.get()) + return prompt_toolkit.ANSI(capture.get().strip()) def row_ib(name): From d1b398318bc944b13689db3487c56a84270e289c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 16 May 2021 20:21:11 -0400 Subject: [PATCH 10/11] add archive content --- setup.cfg | 1 + src/plotman/archive.py | 7 +++- src/plotman/monitor.py | 78 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index c7697bdb..c83e63e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,7 @@ install_requires = pyyaml rich texttable + toolz [options.packages.find] where=src diff --git a/src/plotman/archive.py b/src/plotman/archive.py index 059a48e5..a8cfb306 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -88,7 +88,12 @@ def get_archdir_freebytes(arch_cfg): archdir_freebytes = {} df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' % (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, arch_cfg.rsyncd_path) ) - with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc: + with subprocess.Popen( + df_cmd, + shell=True, + stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + ) as proc: for line in proc.stdout.readlines(): fields = line.split() if fields[3] == b'-': diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index d3a65194..4a5f2da4 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -2,6 +2,7 @@ import functools import io import itertools +import math import os import time @@ -19,6 +20,7 @@ import rich.layout import rich.live import rich.table +import toolz import plotman.archive import plotman.configuration @@ -107,14 +109,18 @@ async def with_prompt_toolkit(): config_text = plotman.configuration.read_configuration_text(config_path) cfg = plotman.configuration.get_validated_configs(config_text, config_path) + archiving_configured = cfg.directories.archive is not None + tmp_prefix = os.path.commonpath(cfg.directories.tmp) dst_prefix = os.path.commonpath(cfg.directories.dst) + if archiving_configured: + arch_prefix = cfg.directories.archive.rsyncd_path header_buffer = prompt_toolkit.layout.controls.FormattedTextControl('header') plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl('plots') tmp_buffer = prompt_toolkit.layout.controls.FormattedTextControl('tmp') dst_buffer = prompt_toolkit.layout.controls.FormattedTextControl('dst') - archive_buffer = prompt_toolkit.layout.controls.FormattedTextControl('archive') + archive_buffer = prompt_toolkit.layout.controls.FormattedTextControl('archive ') logs_buffer = prompt_toolkit.layout.controls.FormattedTextControl('logs') footer_buffer = prompt_toolkit.layout.controls.FormattedTextControl('footer') @@ -136,7 +142,7 @@ async def with_prompt_toolkit(): prompt_toolkit.layout.containers.Window(content=header_buffer, dont_extend_height=True), prompt_toolkit.layout.containers.Window(content=plots_buffer, dont_extend_height=True), prompt_toolkit.layout.containers.VSplit(disk_columns, padding=1), - prompt_toolkit.layout.containers.Window(content=archive_buffer, dont_extend_height=True), + prompt_toolkit.layout.containers.Window(content=archive_buffer, dont_extend_height=True, wrap_lines=True), prompt_toolkit.layout.containers.Window(content=logs_buffer), prompt_toolkit.layout.containers.Window(content=footer_buffer, dont_extend_height=True), ] @@ -160,7 +166,9 @@ async def with_prompt_toolkit(): jobs = [] - key_bindings.add('q', 'c-c')(exit_key_binding) + # i think this should be able to be a single call... + key_bindings.add('q')(exit_key_binding) + key_bindings.add('c-c')(exit_key_binding) binding_handler_names = { exit_key_binding: 'exit', @@ -217,6 +225,20 @@ async def with_prompt_toolkit(): dst_table = build_dst_table(dst_data=dst_data) dst_buffer.text = capture_rich(dst_table, console=rich_console) + size = application.output.get_size() + + if archiving_configured: + archdir_freebytes = plotman.archive.get_archdir_freebytes(cfg.directories.archive) + archdir_rich = arch_dir_text( + archdir_freebytes=archdir_freebytes, + width=size.columns, + prefix=arch_prefix, + ) + archive_buffer.text = capture_rich(archdir_rich, console=rich_console) + + logs_rich = '[reverse]log:[/reverse]' + logs_buffer.text = capture_rich(logs_rich, console=rich_console) + application.invalidate() await anyio.sleep(1) @@ -226,7 +248,7 @@ async def cancel_after_application(application, cancel_scope): cancel_scope.cancel() -async def exit_key_binding(event): +def exit_key_binding(event): event.app.exit() @@ -415,3 +437,51 @@ def build_dst_table(dst_data): table.add_row(*attr.astuple(row)) return table + + +def build_dst_table(dst_data): + table = rich.table.Table(box=None, header_style='reverse') + + for field in attr.fields(DstRow): + table.add_column(field.metadata['name']) + + for row in dst_data: + table.add_row(*attr.astuple(row)) + + return table + + +def arch_dir_text(archdir_freebytes, width, prefix): + lines = [ + '[reverse]archive dirs free space[/reverse]', + ] + + if len(archdir_freebytes) == 0: + lines.append('') + return '\n'.join(lines) + + abbreviated_archdir_freebytes = { + plotman.reporting.abbr_path(path=path, putative_prefix=prefix): plotman.plot_util.human_format(free, 0) + for path, free in archdir_freebytes.items() + } + + maximum_path_length = max(len(path) for path in abbreviated_archdir_freebytes.keys()) + maximum_free_length = max(len(path) for path in abbreviated_archdir_freebytes.values()) + + cells = [ + f'{path: <{maximum_path_length}} - {free: >{maximum_free_length}}' + for path, free in sorted(abbreviated_archdir_freebytes.items()) + ] + + cell_divider = ' | ' + + cells_per_line = math.floor( + (width + len(cell_divider)) / (len(cells[0]) + len(cell_divider)) + ) + + lines.extend( + cell_divider.join(row) + for row in toolz.partition_all(cells_per_line, cells) + ) + + return '\n'.join(lines) From 4fa7142225813ce94c01fd2d3239393f332fd7b6 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 9 Feb 2023 20:42:12 -0500 Subject: [PATCH 11/11] catchup a bit --- mypy.ini | 3 + src/plotman/job.py | 1 + src/plotman/monitor.py | 124 ++++++++++++++++++++++------------------- 3 files changed, 72 insertions(+), 56 deletions(-) diff --git a/mypy.ini b/mypy.ini index 2b2c6b0f..2c75129e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,5 +22,8 @@ ignore_missing_imports = true [mypy-texttable] ignore_missing_imports = true +[mypy-toolz] +ignore_missing_imports = true + [mypy-yaml] ignore_missing_imports = true diff --git a/src/plotman/job.py b/src/plotman/job.py index 4d9dad3f..a715d51b 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -12,6 +12,7 @@ import psutil import plotman.errors +import plotman.plotters if typing.TYPE_CHECKING: import plotman.errors diff --git a/src/plotman/monitor.py b/src/plotman/monitor.py index 4a5f2da4..7d83001a 100644 --- a/src/plotman/monitor.py +++ b/src/plotman/monitor.py @@ -1,10 +1,12 @@ import collections import functools +import importlib import io import itertools import math import os import time +import typing import anyio import attr @@ -28,15 +30,23 @@ import plotman.manager import plotman.plot_util import plotman.reporting +import plotman.resources -def with_rich(): +def with_rich() -> None: config_path = plotman.configuration.get_path() config_text = plotman.configuration.read_configuration_text(config_path) - cfg = plotman.configuration.get_validated_configs(config_text, config_path) + preset_target_definitions_text = importlib.resources.read_text( + plotman.resources, + "target_definitions.yaml", + ) + cfg = plotman.configuration.get_validated_configs(config_text, config_path, preset_target_definitions_text) tmp_prefix = os.path.commonpath(cfg.directories.tmp) - dst_prefix = os.path.commonpath(cfg.directories.dst) + if cfg.directories.dst is None: + dst_prefix = "" + else: + dst_prefix = os.path.commonpath(cfg.directories.dst) overall = rich.layout.Layout('overall') rows = [ @@ -64,7 +74,7 @@ def with_rich(): disks_layout.split_row(*disks_layouts) - jobs = [] + jobs: typing.List[plotman.job.Job] = [] prompt_toolkit_input = prompt_toolkit.input.create_input() with prompt_toolkit_input.raw_mode(): @@ -73,7 +83,7 @@ def with_rich(): header_layout.update(str(i)) jobs = plotman.job.Job.get_running_jobs( - cfg.directories.log, + cfg.logging.plots, cached_jobs=jobs, ) jobs_data = build_jobs_data( @@ -104,17 +114,20 @@ def with_rich(): time.sleep(0.1) -async def with_prompt_toolkit(): +async def with_prompt_toolkit() -> None: config_path = plotman.configuration.get_path() config_text = plotman.configuration.read_configuration_text(config_path) - cfg = plotman.configuration.get_validated_configs(config_text, config_path) - - archiving_configured = cfg.directories.archive is not None + preset_target_definitions_text = importlib.resources.read_text( + plotman.resources, + "target_definitions.yaml", + ) + cfg = plotman.configuration.get_validated_configs(config_text, config_path, preset_target_definitions_text) tmp_prefix = os.path.commonpath(cfg.directories.tmp) - dst_prefix = os.path.commonpath(cfg.directories.dst) - if archiving_configured: - arch_prefix = cfg.directories.archive.rsyncd_path + if cfg.directories.dst is None: + dst_prefix = "" + else: + dst_prefix = os.path.commonpath(cfg.directories.dst) header_buffer = prompt_toolkit.layout.controls.FormattedTextControl('header') plots_buffer = prompt_toolkit.layout.controls.FormattedTextControl('plots') @@ -153,24 +166,24 @@ async def with_prompt_toolkit(): key_bindings = prompt_toolkit.key_binding.KeyBindings() - application = prompt_toolkit.Application( + application = prompt_toolkit.Application[None]( layout=layout, full_screen=True, key_bindings=key_bindings, ) # https://github.com/prompt-toolkit/python-prompt-toolkit/issues/827#issuecomment-459099452 - application.output.show_cursor = lambda: False + application.output.show_cursor = lambda: False # type: ignore[assignment] rich_console = rich.console.Console() - jobs = [] + jobs: typing.List[plotman.job.Job] = [] # i think this should be able to be a single call... key_bindings.add('q')(exit_key_binding) key_bindings.add('c-c')(exit_key_binding) - binding_handler_names = { + binding_handler_names: typing.Dict[typing.Callable[[prompt_toolkit.key_binding.key_processor.KeyPressEvent], object], str] = { exit_key_binding: 'exit', } @@ -194,7 +207,7 @@ async def with_prompt_toolkit(): header_buffer.text = str(i) jobs = plotman.job.Job.get_running_jobs( - cfg.directories.log, + cfg.logging.plots, cached_jobs=jobs, ) jobs_data = build_jobs_data( @@ -227,8 +240,15 @@ async def with_prompt_toolkit(): size = application.output.get_size() - if archiving_configured: - archdir_freebytes = plotman.archive.get_archdir_freebytes(cfg.directories.archive) + if cfg.archiving is not None: + archdir_freebytes, log_message = plotman.archive.get_archdir_freebytes(cfg.archiving) + + archive_directories = list(archdir_freebytes.keys()) + if len(archive_directories) == 0: + arch_prefix = "" + else: + arch_prefix = os.path.commonpath(archive_directories) + archdir_rich = arch_dir_text( archdir_freebytes=archdir_freebytes, width=size.columns, @@ -243,23 +263,23 @@ async def with_prompt_toolkit(): await anyio.sleep(1) -async def cancel_after_application(application, cancel_scope): +async def cancel_after_application(application: prompt_toolkit.Application[None], cancel_scope: anyio.CancelScope) -> None: await application.run_async() cancel_scope.cancel() -def exit_key_binding(event): +def exit_key_binding(event: prompt_toolkit.key_binding.key_processor.KeyPressEvent) -> None: event.app.exit() -def capture_rich(*objects, console): +def capture_rich(*objects: object, console: rich.console.Console) -> prompt_toolkit.ANSI: with console.capture() as capture: console.print(*objects) return prompt_toolkit.ANSI(capture.get().strip()) -def row_ib(name): +def row_ib(name: str) -> typing.Any: return attr.ib(converter=str, metadata={'name': name}) @@ -280,16 +300,17 @@ class JobRow: io: str = row_ib(name='io') @classmethod - def from_job(cls, job, dst_prefix, tmp_prefix): + def from_job(cls, job: plotman.job.Job, dst_prefix: str, tmp_prefix: str) -> "JobRow": + plot_info = job.plotter.common_info() self = cls( - plot_id=job.plot_id[:8], - k=job.k, - tmp_path=plotman.reporting.abbr_path(job.tmpdir, tmp_prefix), - dst=plotman.reporting.abbr_path(job.dstdir, dst_prefix), + plot_id=job.plot_id_prefix(), + k=str(plot_info.plot_size), + tmp_path=plotman.reporting.abbr_path(plot_info.tmpdir, tmp_prefix), + dst=plotman.reporting.abbr_path(plot_info.dstdir, dst_prefix), wall=plotman.plot_util.time_format(job.get_time_wall()), - phase=plotman.reporting.phase_str(job.progress()), + phase=plotman.reporting.phases_str([job.progress()]), tmp_usage=plotman.plot_util.human_format(job.get_tmp_usage(), 0), - pid=job.proc.pid, + pid=str(job.proc.pid), stat=job.get_run_status(), mem=plotman.plot_util.human_format(job.get_mem_usage(), 1), user=plotman.plot_util.time_format(job.get_time_user()), @@ -300,7 +321,7 @@ def from_job(cls, job, dst_prefix, tmp_prefix): return self -def build_jobs_data(jobs, dst_prefix, tmp_prefix): +def build_jobs_data(jobs: typing.List[plotman.job.Job], dst_prefix: str, tmp_prefix: str) -> typing.List[JobRow]: sorted_jobs = sorted(jobs, key=plotman.job.Job.get_time_wall) jobs_data = [ @@ -311,7 +332,7 @@ def build_jobs_data(jobs, dst_prefix, tmp_prefix): return jobs_data -def build_jobs_table(jobs_data): +def build_jobs_table(jobs_data: typing.List[JobRow]) -> rich.table.Table: table = rich.table.Table(box=None, header_style='reverse') table.add_column('#') @@ -329,10 +350,10 @@ def build_jobs_table(jobs_data): class TmpRow: path: str = row_ib(name='tmp') ready: bool = row_ib(name='ready') - phases: list[plotman.job.Phase] = row_ib(name='phases') + phases: str = row_ib(name='phases') @classmethod - def from_tmp(cls, dir_cfg, jobs, sched_cfg, tmp, prefix): + def from_tmp(cls, dir_cfg: plotman.configuration.Directories, jobs: typing.List[plotman.job.Job], sched_cfg: plotman.configuration.Scheduling, tmp: str, prefix: str) -> "TmpRow": phases = sorted(plotman.job.job_phases_for_tmpdir(d=tmp, all_jobs=jobs)) tmp_suffix = plotman.reporting.abbr_path(path=tmp, putative_prefix=prefix) ready = plotman.manager.phases_permit_new_job( @@ -343,13 +364,13 @@ def from_tmp(cls, dir_cfg, jobs, sched_cfg, tmp, prefix): ) self = cls( path=tmp_suffix, - ready='OK' if ready else '--', + ready=ready, phases=plotman.reporting.phases_str(phases=phases, max_num=5), ) return self -def build_tmp_data(jobs, dir_cfg, sched_cfg, prefix): +def build_tmp_data(jobs: typing.List[plotman.job.Job], dir_cfg: plotman.configuration.Directories, sched_cfg: plotman.configuration.Scheduling, prefix: str) -> typing.List[TmpRow]: rows = [ TmpRow.from_tmp( dir_cfg=dir_cfg, @@ -364,7 +385,7 @@ def build_tmp_data(jobs, dir_cfg, sched_cfg, prefix): return rows -def build_tmp_table(tmp_data): +def build_tmp_table(tmp_data: typing.List[TmpRow]) -> rich.table.Table: table = rich.table.Table(box=None, header_style='reverse') for field in attr.fields(TmpRow): @@ -380,18 +401,18 @@ def build_tmp_table(tmp_data): class DstRow: dst: str = row_ib(name='dst') plot_count: int = row_ib(name='plots') - free: int = row_ib(name='free') - inbound_phases: list[plotman.job.Phase] = row_ib(name='phases') + free: str = row_ib(name='free') + inbound_phases: str = row_ib(name='phases') priority: int = row_ib(name='pri') @classmethod - def from_dst(cls, dst, jobs, prefix, dir2oldphase): + def from_dst(cls, dst: str, jobs: typing.List[plotman.job.Job], prefix: str, dir2oldphase: typing.Dict[str, plotman.job.Phase]) -> "DstRow": # TODO: This logic is replicated in archive.py's priority computation, # maybe by moving more of the logic in to directory.py eldest_ph = dir2oldphase.get(dst, plotman.job.Phase(0, 0)) phases = plotman.job.job_phases_for_dstdir(dst, jobs) - dir_plots = plotman.plot_util.list_k32_plots(dst) + dir_plots = plotman.plot_util.list_plots(dst) free = plotman.plot_util.df_b(dst) n_plots = len(dir_plots) priority = plotman.archive.compute_priority( @@ -411,9 +432,12 @@ def from_dst(cls, dst, jobs, prefix, dir2oldphase): return self -def build_dst_data(jobs, dstdirs, prefix): +def build_dst_data(jobs: typing.List[plotman.job.Job], dstdirs: typing.Optional[typing.List[str]], prefix: str) -> typing.List[DstRow]: dir2oldphase = plotman.manager.dstdirs_to_furthest_phase(jobs) + if dstdirs is None: + return [] + rows = [ DstRow.from_dst( dst=dst, @@ -427,19 +451,7 @@ def build_dst_data(jobs, dstdirs, prefix): return rows -def build_dst_table(dst_data): - table = rich.table.Table(box=None, header_style='reverse') - - for field in attr.fields(DstRow): - table.add_column(field.metadata['name']) - - for row in dst_data: - table.add_row(*attr.astuple(row)) - - return table - - -def build_dst_table(dst_data): +def build_dst_table(dst_data: typing.List[DstRow]) -> rich.table.Table: table = rich.table.Table(box=None, header_style='reverse') for field in attr.fields(DstRow): @@ -451,7 +463,7 @@ def build_dst_table(dst_data): return table -def arch_dir_text(archdir_freebytes, width, prefix): +def arch_dir_text(archdir_freebytes: typing.Dict[str, int], width: int, prefix: str) -> str: lines = [ '[reverse]archive dirs free space[/reverse]', ]