Skip to content

Commit ba6043d

Browse files
oliver-sandersoliver-sanders
authored
Merge pull request #6480 from oliver-sanders/cat-log-list-tailable-files
cat-log: list out/err files when available via tailer
2 parents 9fe7f51 + 4714a46 commit ba6043d

File tree

4 files changed

+150
-14
lines changed

4 files changed

+150
-14
lines changed

‎changes.d/6480.fix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
`cat-log`: List log files which are available via a configured tailer/viewer command.

‎cylc/flow/scripts/cat_log.py‎

Lines changed: 37 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -596,28 +596,51 @@ def _main(
596596
cmd.append('--prepend-path')
597597
cmd.append(workflow_id)
598598
# TODO: Add Intelligent Host selection to this
599+
proc = None
599600
with suppress(KeyboardInterrupt):
600601
# (Ctrl-C while tailing)
601602
# NOTE: This will raise NoHostsError if the platform is not
602603
# contactable
603-
remote_cylc_cmd(
604+
proc = remote_cylc_cmd(
604605
cmd,
605606
platform,
606-
capture_process=False,
607+
capture_process=(mode == 'list-dir'),
607608
manage=(mode == 'tail'),
608-
text=False
609+
text=(mode == 'list-dir'),
609610
)
610-
if (
611-
mode == 'list-dir'
612-
and os.path.exists(
613-
os.path.join(
614-
local_log_dir,
615-
'job-activity.log'
616-
)
617-
)
618-
):
619-
# add the local-only job-activity.log file to the remote-list
620-
print('job-activity.log')
611+
612+
# add and missing items to file listing results
613+
if isinstance(proc, Popen):
614+
# i.e: if mode=='list-dir' and ctrl+c not pressed
615+
out, err = proc.communicate()
616+
files = out.splitlines()
617+
618+
# add files which can be accessed via a tailer
619+
if live_job_id is not None:
620+
if (
621+
# NOTE: only list the file if it can be viewed in
622+
# both modes
623+
(platform['out tailer'] and platform['out viewer'])
624+
and 'job.out' not in files
625+
):
626+
files.append('job.out')
627+
if (
628+
(platform['err tailer'] and platform['err viewer'])
629+
and 'job.err' not in files
630+
):
631+
files.append('job.err')
632+
633+
# add the job-activity.log file which is always local
634+
if os.path.exists(
635+
os.path.join(local_log_dir, 'job-activity.log')
636+
):
637+
files.append('job-activity.log')
638+
639+
files.sort()
640+
print('\n'.join(files))
641+
print(err, file=sys.stderr)
642+
sys.exit(proc.returncode)
643+
621644
else:
622645
# Local task job or local job log.
623646
logpath = os.path.join(local_log_dir, options.filename)

‎tests/functional/cylc-cat-log/13-remote-out-err-tailer.t‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env bash
2+
# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
3+
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
#-------------------------------------------------------------------------------
18+
# Test "cylc cat-log" with custom out/err tailers
19+
export REQUIRE_PLATFORM='loc:remote runner:background fs:indep comms:tcp'
20+
. "$(dirname "$0")/test_header"
21+
#-------------------------------------------------------------------------------
22+
set_test_number 12
23+
#-------------------------------------------------------------------------------
24+
# run the workflow
25+
TEST_NAME="${TEST_NAME_BASE}-validate"
26+
install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
27+
run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}"
28+
workflow_run_ok "${TEST_NAME_BASE}-run" cylc play -N "${WORKFLOW_NAME}"
29+
#-------------------------------------------------------------------------------
30+
# change the platform the task ran on to the remote platform
31+
sqlite3 "${HOME}/cylc-run/${WORKFLOW_NAME}/log/db" "
32+
UPDATE
33+
task_jobs
34+
SET
35+
platform_name = '${CYLC_TEST_PLATFORM}',
36+
run_status = null
37+
WHERE
38+
name = 'foo'
39+
AND cycle = '1'
40+
;"
41+
#-------------------------------------------------------------------------------
42+
# test cylc cat-log --mode=list-dir will not list job.out / err
43+
# (no tailer / viewer configured)
44+
create_test_global_config "" "
45+
[platforms]
46+
[[$CYLC_TEST_PLATFORM]]
47+
out tailer =
48+
err tailer =
49+
out viewer =
50+
err viewer =
51+
"
52+
TEST_NAME="${TEST_NAME_BASE}-list-dir-no-tailers"
53+
# NOTE: command will fail due to missing remote directory (this tests remote
54+
# error code is preserved)
55+
run_fail "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -m 'list-dir'
56+
# the job.out and job.err filees
57+
grep_fail "job.out" "${TEST_NAME}.stdout"
58+
grep_fail "job.err" "${TEST_NAME}.stdout"
59+
#-------------------------------------------------------------------------------
60+
# test cylc cat-log --mode=list-dir lists the tailed files
61+
# (both tailer and viewer configured)
62+
create_test_global_config "" "
63+
[platforms]
64+
[[$CYLC_TEST_PLATFORM]]
65+
out tailer = echo OUT
66+
err tailer = echo ERR
67+
out viewer = echo OUT
68+
err viewer = echo ERR
69+
"
70+
# test cylc cat-log --mode=list-dir lists the tailed files
71+
TEST_NAME="${TEST_NAME_BASE}-list-dir-with-tailers"
72+
# NOTE: command will fail due to missing remote directory (this tests remote
73+
# error code is preserved)
74+
run_fail "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -m 'list-dir'
75+
# the job.out and job.err filees
76+
grep_ok "job.out" "${TEST_NAME}.stdout"
77+
grep_ok "job.err" "${TEST_NAME}.stdout"
78+
#-------------------------------------------------------------------------------
79+
# test cylc cat-log runs the custom tailers
80+
TEST_NAME="${TEST_NAME_BASE}-cat-out"
81+
run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f o -m t
82+
grep_ok "OUT" "${TEST_NAME}.stdout"
83+
run_ok "${TEST_NAME}" cylc cat-log "${WORKFLOW_NAME}//1/foo" -f e -m t
84+
grep_ok "ERR" "${TEST_NAME}.stdout"
85+
#-------------------------------------------------------------------------------
86+
purge
87+
exit

‎tests/functional/cylc-cat-log/13-remote-out-err-tailer/flow.cylc‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
[scheduler]
2+
[[events]]
3+
abort on stall timeout = True
4+
stall timeout = PT2M
5+
6+
[scheduling]
7+
[[graph]]
8+
R1 = foo
9+
10+
[runtime]
11+
[[foo]]
12+
script = """
13+
# wait for the started message to be received
14+
cylc__job__poll_grep_workflow_log -E 'foo.*running'
15+
16+
# remove the out/err files
17+
rm "${CYLC_TASK_LOG_DIR}/job.out"
18+
rm "${CYLC_TASK_LOG_DIR}/job.err"
19+
20+
# stop the workflow, orphaning this job
21+
cylc stop --now --now "${CYLC_WORKFLOW_ID}" 2>/dev/null >/dev/null
22+
23+
# suppress any subsequent messages
24+
rm "${CYLC_WORKFLOW_RUN_DIR}/.service/contact"
25+
"""

0 commit comments

Comments
 (0)