Skip to content

Commit 938a21a

Browse files
jay-taujay-tau
authored
feat: Export directory prefix (#16)
* feat: Export directory prefix (Darshan) * format: Remove debug statements * add export_dir support for Recorder * chore: Remove commented-out code for exporting files in handle_recorder.py * feat: Update export file name to include "-summary" * fix: Update export_csv function call in handle_recorder.py * fix: Update darshan dependency to version 3.4.4.0 or higher * fix: Update darshan dependency to version 3.4.4.0 or higher * fmt: Clean up split_files case * fmt: import statements * fmt: Rename `input_filename` to `trace_name`
1 parent 190696a commit 938a21a

File tree

6 files changed

+103
-96
lines changed

6 files changed

+103
-96
lines changed

‎drishti/handlers/handle_darshan.py‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -752,15 +752,10 @@ def handler():
752752
display_thresholds(console)
753753
display_footer(console, insights_start_time, insights_end_time)
754754

755-
filename = '{}.html'.format(args.log_path)
756-
export_html(console, filename)
757-
758-
filename = '{}.svg'.format(args.log_path)
759-
export_svg(console, filename)
760-
761-
filename = '{}-summary.csv'.format(
762-
args.log_path.replace('.darshan', '')
763-
)
764-
765-
export_csv(filename, job['job']['jobid'])
755+
# Export to HTML, SVG, and CSV
756+
trace_name = os.path.basename(args.log_path).replace('.darshan', '')
757+
out_dir = args.export_dir if args.export_dir != "" else os.getcwd()
766758

759+
export_html(console, out_dir, trace_name)
760+
export_svg(console, out_dir, trace_name)
761+
export_csv(out_dir, trace_name, job['job']['jobid'])

‎drishti/handlers/handle_recorder.py‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
import os
44
import time
5+
56
import pandas as pd
67
from recorder_utils import RecorderReader
78
from recorder_utils.build_offset_intervals import build_offset_intervals
9+
810
from drishti.includes.module import *
911

1012

@@ -577,23 +579,12 @@ def process_helper(file_map, df_intervals, df_posix_records, fid=None):
577579
display_thresholds(console)
578580
display_footer(console, insights_start_time, insights_end_time)
579581

582+
# Export to HTML, SVG, and CSV
583+
trace_name = os.path.basename(os.path.dirname(args.log_path))
580584
if args.split_files:
581-
filename = '{}.{}.html'.format(args.log_path, fid)
582-
else:
583-
filename = '{}.html'.format(args.log_path)
584-
585-
export_html(console, filename)
586-
587-
if args.split_files:
588-
filename = '{}.{}.svg'.format(args.log_path, fid)
589-
else:
590-
filename = '{}.svg'.format(args.log_path)
591-
592-
export_svg(console, filename)
593-
594-
if args.split_files:
595-
filename = '{}.{}.summary.csv'.format(args.log_path, fid)
596-
else:
597-
filename = '{}-summary.csv'.format(args.log_path)
598-
export_csv(filename)
585+
trace_name = f"{trace_name}.{fid}"
586+
out_dir = args.export_dir if args.export_dir != "" else os.getcwd()
599587

588+
export_html(console, out_dir, trace_name)
589+
export_svg(console, out_dir, trace_name)
590+
export_csv(out_dir, trace_name)

‎drishti/includes/module.py‎

Lines changed: 78 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1823,76 +1823,90 @@ def display_footer(console, insights_start_time, insights_end_time):
18231823
)
18241824
)
18251825

1826-
def export_html(console, filename):
1827-
'''
1828-
'''
18291826

1830-
if args.export_html:
1831-
console.save_html(
1832-
filename,
1833-
theme=set_export_theme(),
1834-
clear=False
1835-
)
1827+
def export_html(console, export_dir, trace_name):
1828+
if not args.export_html:
1829+
return
18361830

1831+
os.makedirs(export_dir, exist_ok=True) # Ensure export directory exists
1832+
filepath = os.path.join(export_dir, f"{trace_name}.html")
18371833

1838-
def export_svg(console, filename):
1839-
if args.export_svg:
1840-
console.save_svg(
1841-
filename,
1842-
title='Drishti',
1843-
theme=set_export_theme(),
1844-
clear=False
1845-
)
1834+
console.save_html(
1835+
filepath,
1836+
theme=set_export_theme(),
1837+
clear=False
1838+
)
18461839

18471840

1848-
def export_csv(filename, jobid=None):
1849-
if args.export_csv:
1850-
issues = [
1851-
'JOB',
1852-
INSIGHTS_STDIO_HIGH_USAGE,
1853-
INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
1854-
INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
1855-
INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
1856-
INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
1857-
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
1858-
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
1859-
INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
1860-
INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
1861-
INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
1862-
INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
1863-
INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
1864-
INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
1865-
INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
1866-
INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
1867-
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
1868-
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
1869-
INSIGHTS_POSIX_HIGH_METADATA_TIME,
1870-
INSIGHTS_POSIX_SIZE_IMBALANCE,
1871-
INSIGHTS_POSIX_TIME_IMBALANCE,
1872-
INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
1873-
INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
1874-
INSIGHTS_MPI_IO_NO_USAGE,
1875-
INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
1876-
INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
1877-
INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
1878-
INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
1879-
INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
1880-
INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
1881-
INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
1882-
INSIGHTS_MPI_IO_AGGREGATORS_INTER,
1883-
INSIGHTS_MPI_IO_AGGREGATORS_OK
1884-
]
1885-
if codes:
1886-
issues.extend(codes)
1841+
def export_svg(console, export_dir, trace_name):
1842+
if not args.export_svg:
1843+
return
1844+
1845+
os.makedirs(export_dir, exist_ok=True) # Ensure export directory exists
1846+
filepath = os.path.join(export_dir, f"{trace_name}.svg")
1847+
1848+
console.save_svg(
1849+
filepath,
1850+
title='Drishti',
1851+
theme=set_export_theme(),
1852+
clear=False
1853+
)
18871854

1888-
detected_issues = dict.fromkeys(issues, False)
1889-
detected_issues['JOB'] = jobid
18901855

1891-
for report in csv_report:
1892-
detected_issues[report] = True
1856+
def export_csv(export_dir, trace_name, jobid=None):
1857+
if not args.export_csv:
1858+
return
1859+
1860+
issues = [
1861+
'JOB',
1862+
INSIGHTS_STDIO_HIGH_USAGE,
1863+
INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE,
1864+
INSIGHTS_POSIX_READ_COUNT_INTENSIVE,
1865+
INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE,
1866+
INSIGHTS_POSIX_READ_SIZE_INTENSIVE,
1867+
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE,
1868+
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE,
1869+
INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE,
1870+
INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE,
1871+
INSIGHTS_POSIX_REDUNDANT_READ_USAGE,
1872+
INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE,
1873+
INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE,
1874+
INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE,
1875+
INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE,
1876+
INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE,
1877+
INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE,
1878+
INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE,
1879+
INSIGHTS_POSIX_HIGH_METADATA_TIME,
1880+
INSIGHTS_POSIX_SIZE_IMBALANCE,
1881+
INSIGHTS_POSIX_TIME_IMBALANCE,
1882+
INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE,
1883+
INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE,
1884+
INSIGHTS_MPI_IO_NO_USAGE,
1885+
INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE,
1886+
INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE,
1887+
INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE,
1888+
INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE,
1889+
INSIGHTS_MPI_IO_BLOCKING_READ_USAGE,
1890+
INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE,
1891+
INSIGHTS_MPI_IO_AGGREGATORS_INTRA,
1892+
INSIGHTS_MPI_IO_AGGREGATORS_INTER,
1893+
INSIGHTS_MPI_IO_AGGREGATORS_OK
1894+
]
1895+
if codes:
1896+
issues.extend(codes)
1897+
1898+
detected_issues = dict.fromkeys(issues, False)
1899+
detected_issues['JOB'] = jobid
1900+
1901+
for report in csv_report:
1902+
detected_issues[report] = True
1903+
1904+
1905+
os.makedirs(export_dir, exist_ok=True) # Ensure export directory exists
1906+
filepath = os.path.join(export_dir, f"{trace_name}.csv")
18931907

1894-
with open(filename, 'w') as f:
1895-
w = csv.writer(f)
1896-
w.writerow(detected_issues.keys())
1897-
w.writerow(detected_issues.values())
1908+
with open(filepath, 'w') as f:
1909+
w = csv.writer(f)
1910+
w.writerow(detected_issues.keys())
1911+
w.writerow(detected_issues.values())
18981912

‎drishti/includes/parser.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,13 @@
9696
help='Export a CSV with the code of all issues that were triggered'
9797
)
9898

99+
parser.add_argument(
100+
'--export_dir',
101+
default="",
102+
dest='export_dir',
103+
help='Specify the directory prefix for the output files (if any)'
104+
)
105+
99106
parser.add_argument(
100107
'--json',
101108
default=False,

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
argparse
2-
darshan==3.4.4.0
2+
darshan>=3.4.4.0
33
pandas
44
rich==12.5.1
55
recorder-utils

‎setup.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from setuptools import setup, find_packages
1+
from setuptools import find_packages, setup
22

33
with open("README.md", "r") as f:
44
long_description = f.read()
@@ -19,7 +19,7 @@
1919
install_requires=[
2020
'argparse',
2121
'pandas',
22-
'darshan==3.4.4.0',
22+
'darshan>=3.4.4.0',
2323
'rich==12.5.1',
2424
'recorder-utils',
2525
],

0 commit comments

Comments
 (0)