Skip to content

Commit 5fe6d8e

Browse files
bvliucopybara-github
bvliu
authored and
copybara-github
committed
Split log collector functions into new module.
PiperOrigin-RevId: 823150046
1 parent 4cb87a4 commit 5fe6d8e

File tree

4 files changed

+158
-136
lines changed

4 files changed

+158
-136
lines changed

‎perfkitbenchmarker/linux_benchmarks/cluster_boot_benchmark.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
from perfkitbenchmarker import configs
7676
from perfkitbenchmarker import errors
7777
from perfkitbenchmarker import linux_virtual_machine
78-
from perfkitbenchmarker import log_util
78+
from perfkitbenchmarker import log_collector
7979
from perfkitbenchmarker import sample
8080
from perfkitbenchmarker import virtual_machine
8181
from perfkitbenchmarker import vm_util
@@ -607,7 +607,7 @@ def Cleanup(benchmark_spec):
607607
logging.warning('tcpdump process %s ended prematurely.', pid)
608608
try:
609609
tcpdump_path = benchmark_spec.config.temporary['tcpdump_output_path']
610-
log_util.CollectVMLogs(FLAGS.run_uri, tcpdump_path)
610+
log_collector.CollectVMLogs(FLAGS.run_uri, tcpdump_path)
611611
os.remove(tcpdump_path)
612612
except FileNotFoundError:
613613
logging.warning(

‎perfkitbenchmarker/log_collector.py‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Copyright 2014 PerfKitBenchmarker Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
"""Utilities for collecting logs."""
15+
16+
import datetime
17+
from absl import flags
18+
from perfkitbenchmarker import vm_util
19+
20+
GSUTIL_MV = 'mv'
21+
GSUTIL_CP = 'cp'
22+
GSUTIL_OPERATIONS = [GSUTIL_MV, GSUTIL_CP]
23+
24+
PKB_LOG_BUCKET = flags.DEFINE_string(
25+
'pkb_log_bucket',
26+
None,
27+
'Name of the GCS bucket that PKB logs should route to. If this is not '
28+
'specified, then PKB logs will remain on the VM. This bucket must exist '
29+
'and the caller must have write permissions on the bucket for a successful '
30+
'export.',
31+
)
32+
VM_LOG_BUCKET = flags.DEFINE_string(
33+
'vm_log_bucket',
34+
None,
35+
'The GCS bucket to store VM logs in. If not provided, VM logs will go to '
36+
'the calling machine only. This only applies if --capture_vm_logs is '
37+
'set.',
38+
)
39+
_SAVE_LOG_TO_BUCKET_OPERATION = flags.DEFINE_enum(
40+
'save_log_to_bucket_operation',
41+
GSUTIL_MV,
42+
GSUTIL_OPERATIONS,
43+
'How to save the log to the bucket, available options are mv, cp',
44+
)
45+
_RELATIVE_GCS_LOG_PATH = flags.DEFINE_string(
46+
'relative_gcs_log_path',
47+
None,
48+
'The relative path inside the GCS bucket where to save the log, e.g. '
49+
'"root_dir/sub_dir", and the full file path would be '
50+
'gs://<bucket>/<relative_gcs_log_path>',
51+
)
52+
53+
54+
def CollectPKBLogs(run_uri: str, log_local_path: str) -> None:
55+
"""Move PKB log files over to a GCS bucket (`pkb_log_bucket` flag).
56+
57+
All failures in the process of log collection are suppressed to avoid causing
58+
a run to fail unnecessarily.
59+
60+
Args:
61+
run_uri: The run URI of the benchmark run.
62+
log_local_path: Path to local log file.
63+
"""
64+
if PKB_LOG_BUCKET.value:
65+
# Generate the log path to the cloud bucket based on the invocation date of
66+
# this function.
67+
gcs_log_path = GetLogCloudPath(PKB_LOG_BUCKET.value, f'{run_uri}-pkb.log')
68+
vm_util.IssueCommand(
69+
[
70+
'gsutil',
71+
'-h',
72+
'Content-Type:text/plain',
73+
_SAVE_LOG_TO_BUCKET_OPERATION.value,
74+
'-Z',
75+
log_local_path,
76+
gcs_log_path,
77+
],
78+
raise_on_failure=False,
79+
raise_on_timeout=False,
80+
)
81+
82+
83+
def CollectVMLogs(run_uri: str, source_path: str) -> None:
84+
"""Move VM log files over to a GCS bucket (`vm_log_bucket` flag).
85+
86+
All failures in the process of log collection are suppressed to avoid causing
87+
a run to fail unnecessarily.
88+
89+
Args:
90+
run_uri: The run URI of the benchmark run.
91+
source_path: The path to the log file.
92+
"""
93+
if VM_LOG_BUCKET.value:
94+
source_filename = source_path.split('/')[-1]
95+
gcs_directory_path = GetLogCloudPath(VM_LOG_BUCKET.value, run_uri)
96+
gcs_path = f'{gcs_directory_path}/{source_filename}'
97+
vm_util.IssueCommand(
98+
[
99+
'gsutil',
100+
'-h',
101+
'Content-Type:text/plain',
102+
'mv',
103+
'-Z',
104+
source_path,
105+
gcs_path,
106+
],
107+
raise_on_failure=False,
108+
raise_on_timeout=False,
109+
)
110+
111+
112+
def GetLogCloudPath(log_bucket: str, path_suffix: str) -> str:
113+
"""Returns the GCS path, to where the logs should be saved.
114+
115+
Args:
116+
log_bucket: The GCS bucket to save the logs to.
117+
path_suffix: The suffix to append to the GCS path.
118+
119+
Returns:
120+
The GCS path to where the logs should be saved.
121+
"""
122+
run_date = datetime.date.today()
123+
gcs_path_prefix = _GetGcsPathPrefix(log_bucket)
124+
return (
125+
f'{gcs_path_prefix}/'
126+
+ f'{run_date.year:04d}/{run_date.month:02d}/'
127+
+ f'{run_date.day:02d}/'
128+
+ path_suffix
129+
)
130+
131+
132+
def _GetGcsPathPrefix(bucket: str) -> str:
133+
"""Returns the GCS path prefix, to where the logs should be saved.
134+
135+
Args:
136+
bucket: The GCS bucket to save the logs to.
137+
138+
Returns:
139+
The GCS path prefix, to where the logs should be saved. If
140+
`relative_gcs_log_path` is specified, the prefix will be
141+
gs://<bucket>/<relative_gcs_log_path>. Otherwise, it will be gs://<bucket>.
142+
"""
143+
if _RELATIVE_GCS_LOG_PATH.value:
144+
return f'gs://{bucket}/{_RELATIVE_GCS_LOG_PATH.value}'
145+
return f'gs://{bucket}'

‎perfkitbenchmarker/log_util.py‎

Lines changed: 0 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,11 @@
1414
"""Utilities related to loggers and logging."""
1515

1616
from contextlib import contextmanager
17-
import datetime
1817
import logging
1918
from logging import handlers
2019
import sys
2120
import threading
2221
from absl import flags
23-
from perfkitbenchmarker import vm_util
2422

2523
try:
2624
import colorlog
@@ -43,43 +41,11 @@
4341
log_local_path = None
4442
LOG_FILE_NAME = 'pkb.log'
4543

46-
GSUTIL_MV = 'mv'
47-
GSUTIL_CP = 'cp'
48-
GSUTIL_OPERATIONS = [GSUTIL_MV, GSUTIL_CP]
49-
5044
DEFAULT_LOG_ROTATING_INTERVAL = 1
5145
DEFAULT_LOG_ROTATING_UNIT = 'D'
5246
DEFAULT_LOG_ROTATING_BACKUP_COUNT = 5
5347

5448

55-
PKB_LOG_BUCKET = flags.DEFINE_string(
56-
'pkb_log_bucket',
57-
None,
58-
'Name of the GCS bucket that PKB logs should route to. If this is not '
59-
'specified, then PKB logs will remain on the VM. This bucket must exist '
60-
'and the caller must have write permissions on the bucket for a successful '
61-
'export.',
62-
)
63-
VM_LOG_BUCKET = flags.DEFINE_string(
64-
'vm_log_bucket',
65-
None,
66-
'The GCS bucket to store VM logs in. If not provided, VM logs will go to '
67-
'the calling machine only. This only applies if --capture_vm_logs is '
68-
'set.',
69-
)
70-
_SAVE_LOG_TO_BUCKET_OPERATION = flags.DEFINE_enum(
71-
'save_log_to_bucket_operation',
72-
GSUTIL_MV,
73-
GSUTIL_OPERATIONS,
74-
'How to save the log to the bucket, available options are mv, cp',
75-
)
76-
_RELATIVE_GCS_LOG_PATH = flags.DEFINE_string(
77-
'relative_gcs_log_path',
78-
None,
79-
'The relative path inside the GCS bucket where to save the log, e.g. '
80-
'"root_dir/sub_dir", and the full file path would be '
81-
'gs://<bucket>/<relative_gcs_log_path>',
82-
)
8349
flags.DEFINE_enum(
8450
'log_level',
8551
INFO,
@@ -255,95 +221,3 @@ def ConfigureLogging(
255221
logger.addHandler(handler)
256222
logging.getLogger('requests').setLevel(logging.ERROR)
257223

258-
259-
def CollectPKBLogs(run_uri: str) -> None:
260-
"""Move PKB log files over to a GCS bucket (`pkb_log_bucket` flag).
261-
262-
All failures in the process of log collection are suppressed to avoid causing
263-
a run to fail unnecessarily.
264-
265-
Args:
266-
run_uri: The run URI of the benchmark run.
267-
"""
268-
if PKB_LOG_BUCKET.value:
269-
# Generate the log path to the cloud bucket based on the invocation date of
270-
# this function.
271-
gcs_log_path = GetLogCloudPath(PKB_LOG_BUCKET.value, f'{run_uri}-pkb.log')
272-
vm_util.IssueCommand(
273-
[
274-
'gsutil',
275-
'-h',
276-
'Content-Type:text/plain',
277-
_SAVE_LOG_TO_BUCKET_OPERATION.value,
278-
'-Z',
279-
log_local_path,
280-
gcs_log_path,
281-
],
282-
raise_on_failure=False,
283-
raise_on_timeout=False,
284-
)
285-
286-
287-
def CollectVMLogs(run_uri: str, source_path: str) -> None:
288-
"""Move VM log files over to a GCS bucket (`vm_log_bucket` flag).
289-
290-
All failures in the process of log collection are suppressed to avoid causing
291-
a run to fail unnecessarily.
292-
293-
Args:
294-
run_uri: The run URI of the benchmark run.
295-
source_path: The path to the log file.
296-
"""
297-
if VM_LOG_BUCKET.value:
298-
source_filename = source_path.split('/')[-1]
299-
gcs_directory_path = GetLogCloudPath(VM_LOG_BUCKET.value, run_uri)
300-
gcs_path = f'{gcs_directory_path}/{source_filename}'
301-
vm_util.IssueCommand(
302-
[
303-
'gsutil',
304-
'-h',
305-
'Content-Type:text/plain',
306-
'mv',
307-
'-Z',
308-
source_path,
309-
gcs_path,
310-
],
311-
raise_on_failure=False,
312-
raise_on_timeout=False,
313-
)
314-
315-
316-
def GetLogCloudPath(log_bucket: str, path_suffix: str) -> str:
317-
"""Returns the GCS path, to where the logs should be saved.
318-
319-
Args:
320-
log_bucket: The GCS bucket to save the logs to.
321-
path_suffix: The suffix to append to the GCS path.
322-
323-
Returns:
324-
The GCS path to where the logs should be saved.
325-
"""
326-
run_date = datetime.date.today()
327-
gcs_path_prefix = _GetGcsPathPrefix(log_bucket)
328-
return (
329-
f'{gcs_path_prefix}/'
330-
+ f'{run_date.year:04d}/{run_date.month:02d}/'
331-
+ f'{run_date.day:02d}/'
332-
+ path_suffix
333-
)
334-
335-
336-
def _GetGcsPathPrefix(bucket: str) -> str:
337-
"""Returns the GCS path prefix, to where the logs should be saved.
338-
339-
Args:
340-
bucket: The GCS bucket to save the logs to.
341-
342-
Returns:
343-
The GCS path prefix, to where the logs should be saved. If
344-
`relative_gcs_log_path` is specified, the prefix will be
345-
gs://<bucket>/<relative_gcs_log_path>. Otherwise, it will be gs://<bucket>.
346-
"""
347-
if _RELATIVE_GCS_LOG_PATH.value:
348-
return f'gs://{bucket}/{_RELATIVE_GCS_LOG_PATH.value}'
349-
return f'gs://{bucket}'

‎perfkitbenchmarker/pkb.py‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
from perfkitbenchmarker import flags as pkb_flags
9292
from perfkitbenchmarker import linux_benchmarks
9393
from perfkitbenchmarker import linux_virtual_machine
94+
from perfkitbenchmarker import log_collector
9495
from perfkitbenchmarker import log_util
9596
from perfkitbenchmarker import os_types
9697
from perfkitbenchmarker import package_lookup
@@ -1079,13 +1080,13 @@ def _PublishRunStartedSample(spec):
10791080
"""
10801081
metadata = {'flags': str(flag_util.GetProvidedCommandLineFlags())}
10811082
# Publish the path to this spec's PKB logs at the start of the runs.
1082-
if log_util.PKB_LOG_BUCKET.value and FLAGS.run_uri:
1083-
metadata['pkb_log_path'] = log_util.GetLogCloudPath(
1084-
log_util.PKB_LOG_BUCKET.value, f'{FLAGS.run_uri}-pkb.log'
1083+
if log_collector.PKB_LOG_BUCKET.value and FLAGS.run_uri:
1084+
metadata['pkb_log_path'] = log_collector.GetLogCloudPath(
1085+
log_collector.PKB_LOG_BUCKET.value, f'{FLAGS.run_uri}-pkb.log'
10851086
)
1086-
if log_util.VM_LOG_BUCKET.value and FLAGS.run_uri:
1087-
metadata['vm_log_path'] = log_util.GetLogCloudPath(
1088-
log_util.VM_LOG_BUCKET.value, FLAGS.run_uri
1087+
if log_collector.VM_LOG_BUCKET.value and FLAGS.run_uri:
1088+
metadata['vm_log_path'] = log_collector.GetLogCloudPath(
1089+
log_collector.VM_LOG_BUCKET.value, FLAGS.run_uri
10891090
)
10901091

10911092
_PublishEventSample(spec, 'Run Started', metadata)
@@ -1810,7 +1811,9 @@ def RunBenchmarks():
18101811
_WriteCompletionStatusFile(benchmark_specs, status_file)
18111812

18121813
# Upload PKB logs to GCS after all benchmark runs are complete.
1813-
log_util.CollectPKBLogs(run_uri=FLAGS.run_uri)
1814+
log_collector.CollectPKBLogs(
1815+
run_uri=FLAGS.run_uri, log_local_path=log_util.log_local_path
1816+
)
18141817
all_benchmarks_succeeded = all(
18151818
spec.status == benchmark_status.SUCCEEDED for spec in benchmark_specs
18161819
)
@@ -2054,7 +2057,7 @@ def CaptureVMLogs(
20542057
'Captured the following logs for VM %s: %s', vm.name, vm_log_files
20552058
)
20562059
for log_path in vm_log_files:
2057-
log_util.CollectVMLogs(FLAGS.run_uri, log_path)
2060+
log_collector.CollectVMLogs(FLAGS.run_uri, log_path)
20582061

20592062

20602063
def ParseArgs(argv):

0 commit comments

Comments
 (0)