Skip to content

Commit d62d8b0

Browse files
committed
CBPS-984: collect logs after tools tests
- Set log_file option on all cbimport/cbexport tests - Enable cbimport/cbexport logging by supplying --verbose alongside --log-file - Don't run cbbackupmgr collect-logs redundantly for cbimport/cbexport tests - Run cbbackupmgr collect-logs on all cbbackupmgr tests (most of them were already doing it) Change-Id: Iceca27da8ef4c006a14f8a703323a8d536bb44f2 Reviewed-on: https://review.couchbase.org/c/perfrunner/+/169454 Tested-by: Build Bot <[email protected]> Reviewed-by: Korrigan Clark <[email protected]>
1 parent 293f5ba commit d62d8b0

16 files changed

+46
-21
lines changed

perfrunner/helpers/local.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ def cbexport(master_node: str, cluster_spec: ClusterSpec, bucket: str,
329329
'--scope-field {}'.format(scope_field),
330330
'--threads {}'.format(threads) if threads else None,
331331
'--include-key key',
332-
'--log-file {}'.format(log_file) if log_file else None]
332+
'--verbose --log-file {}'.format(log_file) if log_file else None]
333333

334334
else:
335335
flags = ['--format {}'.format(data_format),
@@ -340,7 +340,7 @@ def cbexport(master_node: str, cluster_spec: ClusterSpec, bucket: str,
340340
'--password {}'.format(cluster_spec.rest_credentials[1]),
341341
'--threads {}'.format(threads) if threads else None,
342342
'--include-key {}'.format(key_field) if key_field else None,
343-
'--log-file {}'.format(log_file) if log_file else None]
343+
'--verbose --log-file {}'.format(log_file) if log_file else None]
344344

345345
cmd = './opt/couchbase/bin/cbexport json {}'.format(
346346
' '.join(filter(None, flags)))
@@ -381,7 +381,7 @@ def cbimport(master_node: str, cluster_spec: ClusterSpec, bucket: str,
381381
'--infer-types' if infer_types else None,
382382
'--omit-empty' if omit_empty else None,
383383
'--errors-log {}'.format(errors_log) if errors_log else None,
384-
'--log-file {}'.format(log_file) if log_file else None]
384+
'--verbose --log-file {}'.format(log_file) if log_file else None]
385385

386386
else:
387387

@@ -401,7 +401,7 @@ def cbimport(master_node: str, cluster_spec: ClusterSpec, bucket: str,
401401
'--infer-types' if infer_types else None,
402402
'--omit-empty' if omit_empty else None,
403403
'--errors-log {}'.format(errors_log) if errors_log else None,
404-
'--log-file {}'.format(log_file) if log_file else None,
404+
'--verbose --log-file {}'.format(log_file) if log_file else None,
405405
'--scope-collection-exp {}'.format(scope_collection_exp)]
406406

407407
cmd = './opt/couchbase/bin/cbimport {} {}'.format(

perfrunner/tests/tools.py

Lines changed: 28 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,11 @@ def run(self):
149149
self.check_num_items()
150150
self.compact_bucket(wait=True)
151151

152-
time_elapsed = self.backup()
152+
try:
153+
time_elapsed = self.backup()
154+
finally:
155+
self.collectinfo()
156+
153157
self.report_kpi(time_elapsed)
154158

155159

@@ -232,7 +236,10 @@ def run(self):
232236

233237
self.access_bg()
234238

235-
time_elapsed = self.backup()
239+
try:
240+
time_elapsed = self.backup()
241+
finally:
242+
self.collectinfo()
236243

237244
self.report_kpi(time_elapsed)
238245

@@ -433,7 +440,12 @@ def run(self):
433440

434441
self.backup()
435442
self.flush_buckets()
436-
time_elapsed = self.restore()
443+
444+
try:
445+
time_elapsed = self.restore()
446+
finally:
447+
self.collectinfo()
448+
437449
self.report_kpi(time_elapsed)
438450

439451

@@ -530,10 +542,7 @@ def export(self):
530542
def run(self):
531543
super().run()
532544

533-
try:
534-
time_elapsed = self.export()
535-
finally:
536-
self.collectinfo()
545+
time_elapsed = self.export()
537546

538547
self.report_kpi(time_elapsed)
539548

@@ -552,10 +561,7 @@ def run(self):
552561

553562
self.flush_buckets()
554563

555-
try:
556-
time_elapsed = self.import_data()
557-
finally:
558-
self.collectinfo()
564+
time_elapsed = self.import_data()
559565

560566
self.report_kpi(time_elapsed)
561567

@@ -571,10 +577,7 @@ def run(self):
571577
self.extract_tools()
572578
self.get_tool_versions()
573579

574-
try:
575-
time_elapsed = self.import_data()
576-
finally:
577-
self.collectinfo()
580+
time_elapsed = self.import_data()
578581

579582
self.report_kpi(time_elapsed)
580583

@@ -633,7 +636,11 @@ def run(self):
633636
if self.test_config.compaction.bucket_compaction == 'true':
634637
self.compact_bucket(wait=True)
635638

636-
time_elapsed = self.backup()
639+
try:
640+
time_elapsed = self.backup()
641+
finally:
642+
self.extract_tools()
643+
self.collectinfo()
637644

638645
self.report_kpi(time_elapsed)
639646

@@ -707,6 +714,10 @@ def run(self):
707714

708715
self.flush_buckets()
709716

710-
time_elapsed = self.restore()
717+
try:
718+
time_elapsed = self.restore()
719+
finally:
720+
self.extract_tools()
721+
self.collectinfo()
711722

712723
self.report_kpi(time_elapsed)

scripts/upload_info.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash +x
22

33
for FILE in *.zip; do
4+
[ -f "$FILE" ] || continue
45
env/bin/aws s3 cp --quiet ${FILE} s3://perf-artifacts/${BUILD_TAG}/${FILE}
56
echo "https://s3-us-west-2.amazonaws.com/perf-artifacts/${BUILD_TAG}/${FILE}"
67
done

tests/collections/tools/tools_export_60M_json_lines_1s_1000c.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,4 @@ threads = 16
3131
format = lines
3232
collection_field = collection
3333
scope_field = scope
34+
log_file = cbexport.log

tests/collections/tools/tools_export_60M_json_lines_1s_1c.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,4 @@ threads = 16
3131
format = lines
3232
collection_field = collection
3333
scope_field = scope
34+
log_file = cbexport.log

tests/collections/tools/tools_import_3M_20K_json_list_1s_1c.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ type = json
3333
collection_field = collection
3434
scope_field = scope
3535
scope_collection_exp = scope-1.collection-1
36+
log_file = cbimport.log

tests/collections/tools/tools_import_60M_json_lines_1s_1c.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,4 @@ type = json
3333
collection_field = collection
3434
scope_field = scope
3535
scope_collection_exp = scope-1.collection-1
36+
log_file = cbimport.log

tests/tools_export_60M_json_lines.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ doc_gen = import_export_simple
2626
[export]
2727
threads = 16
2828
format = lines
29+
log_file = cbexport.log
2930

3031
[clients]
3132
libcouchbase = 2.9.3

tests/tools_export_60M_json_list.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ doc_gen = import_export_simple
2626
[export]
2727
threads = 16
2828
format = list
29+
log_file = cbexport.log
2930

3031
[clients]
3132
libcouchbase = 2.9.3

tests/tools_import_3M_20K_json_list.test

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ doc_gen = import_export_simple
2727
threads = 16
2828
format = list
2929
type = json
30+
log_file = cbimport.log
3031

3132
[clients]
3233
libcouchbase = 2.9.3

0 commit comments

Comments
 (0)