Skip to content

Commit fdcd888

Browse files
authored
Merge branch 'main' into drishti-v0.8
2 parents 8767c07 + c95e628 commit fdcd888

9 files changed

Lines changed: 182 additions & 55 deletions

File tree

.github/workflows/darshan-3.4.2.yml

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
fail-fast: false
1616
matrix:
1717
os: ["ubuntu-latest", "macos-latest"]
18-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
18+
python-version: ["3.9", "3.10", "3.11"]
1919

2020
timeout-minutes: 20
2121

@@ -41,7 +41,7 @@ jobs:
4141
- name: Build Darshan
4242
run: |
4343
cd darshan
44-
git checkout darshan-3.4.0
44+
git checkout darshan-3.4.2
4545
bash prepare.sh
4646
cd darshan-util
4747
./configure
@@ -57,7 +57,7 @@ jobs:
5757
run: |
5858
python -m ensurepip --upgrade
5959
pip install --upgrade setuptools
60-
pip install darshan==3.4.0.1
60+
pip install darshan==3.4.2
6161
pip install -r requirements.txt
6262
pip install .
6363
@@ -125,10 +125,6 @@ jobs:
125125
run: |
126126
dxt-explorer --debug --start 3.7 --end 3.9 --from 1 --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
127127
128-
- name: Setup upterm session
129-
if: ${{ failure() }}
130-
uses: owenthereal/action-upterm@v1
131-
132128
- name: Upload Artifact
133129
if: always()
134130
uses: actions/upload-artifact@v4

.github/workflows/darshan-3.4.4.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
fail-fast: false
1616
matrix:
1717
os: ["ubuntu-latest", "macos-latest"]
18-
python-version: ["3.8", "3.9", "3.10", "3.11"]
18+
python-version: ["3.9", "3.10", "3.11"]
1919

2020
timeout-minutes: 20
2121

@@ -125,10 +125,6 @@ jobs:
125125
run: |
126126
dxt-explorer --debug --start 3.7 --end 3.9 --from 1 --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
127127
128-
- name: Setup upterm session
129-
if: ${{ failure() }}
130-
uses: owenthereal/action-upterm@v1
131-
132128
- name: Upload Artifact
133129
if: always()
134130
uses: actions/upload-artifact@v4

.github/workflows/darshan-3.4.6.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
fail-fast: false
1616
matrix:
1717
os: ["ubuntu-latest", "macos-latest"]
18-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
18+
python-version: ["3.9", "3.10", "3.11", "3.12"]
1919

2020
timeout-minutes: 20
2121

@@ -125,10 +125,6 @@ jobs:
125125
run: |
126126
dxt-explorer --debug --start 3.7 --end 3.9 --from 1 --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
127127
128-
- name: Setup upterm session
129-
if: ${{ failure() }}
130-
uses: owenthereal/action-upterm@v1
131-
132128
- name: Upload Artifact
133129
if: always()
134130
uses: actions/upload-artifact@v4

.github/workflows/darshan-3.4.7.yml

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
fail-fast: false
1616
matrix:
1717
os: ["ubuntu-latest", "macos-latest"]
18-
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
18+
python-version: ["3.9", "3.10", "3.11", "3.12"]
1919

2020
timeout-minutes: 20
2121

@@ -125,10 +125,6 @@ jobs:
125125
run: |
126126
dxt-explorer --debug --start 3.7 --end 3.9 --from 1 --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
127127
128-
- name: Setup upterm session
129-
if: ${{ failure() }}
130-
uses: owenthereal/action-upterm@v1
131-
132128
- name: Upload Artifact
133129
if: always()
134130
uses: actions/upload-artifact@v4
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
name: DXT Explorer (Darshan 3.5.0)
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
explorer:
13+
runs-on: ${{ matrix.os }}
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
os: ["ubuntu-latest", "macos-latest"]
18+
python-version: ["3.11", "3.12", "3.13", "3.14"]
19+
20+
timeout-minutes: 20
21+
22+
steps:
23+
- uses: actions/checkout@v4
24+
with:
25+
submodules: true
26+
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Dependencies
33+
run: |
34+
git clone https://github.com/darshan-hpc/darshan.git
35+
36+
- name: Install macOS Dependencies
37+
if: runner.os == 'macOS'
38+
run: |
39+
brew install libtool autoconf automake
40+
41+
- name: Build Darshan
42+
run: |
43+
cd darshan
44+
git checkout 3.5.0
45+
bash prepare.sh
46+
cd darshan-util
47+
./configure
48+
make
49+
sudo make install
50+
51+
- name: Configure Linux
52+
if: runner.os == 'Linux'
53+
run: |
54+
sudo ldconfig
55+
56+
- name: Install DXT Explorer
57+
run: |
58+
python -m ensurepip --upgrade
59+
pip install --upgrade setuptools
60+
pip install darshan==3.5.0
61+
pip install -r requirements.txt
62+
pip install .
63+
64+
- name: Run DXT Explorer (help)
65+
run: |
66+
dxt-explorer -h
67+
68+
- name: Run DXT Explorer (list)
69+
run: |
70+
dxt-explorer --debug --list sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
71+
72+
- name: Run DXT Explorer (operation)
73+
run: |
74+
dxt-explorer --debug sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
75+
76+
- name: Run DXT Explorer (transfer)
77+
run: |
78+
dxt-explorer --debug --transfer sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
79+
80+
- name: Run DXT Explorer (spatiality)
81+
run: |
82+
dxt-explorer --debug --spatiality sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
83+
84+
- name: Run DXT Explorer (io phase)
85+
run: |
86+
dxt-explorer --debug --io_phase sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
87+
88+
- name: Run DXT Explorer (ost usage operation)
89+
run: |
90+
dxt-explorer --debug --ost_usage_operation sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
91+
92+
- name: Run DXT Explorer (ost usage transfer)
93+
run: |
94+
dxt-explorer --debug --ost_usage_transfer sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
95+
96+
- name: Run DXT Explorer (rank zero workload)
97+
run: |
98+
dxt-explorer --debug --rank_zero_workload sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
99+
100+
- name: Run DXT Explorer (unbalanced workload)
101+
run: |
102+
dxt-explorer --debug --unbalanced_workload sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
103+
104+
- name: Run DXT Explorer (stragglers)
105+
run: |
106+
dxt-explorer --debug --stragglers sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
107+
108+
- name: Run DXT Explorer (truncate runtime)
109+
run: |
110+
dxt-explorer --debug --start 3.7 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
111+
112+
- name: Run DXT Explorer (truncate runtime)
113+
run: |
114+
dxt-explorer --debug --end 3.9 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
115+
116+
- name: Run DXT Explorer (truncate rank)
117+
run: |
118+
dxt-explorer --debug --from 1 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
119+
120+
- name: Run DXT Explorer (truncate rank)
121+
run: |
122+
dxt-explorer --debug --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
123+
124+
- name: Run DXT Explorer (truncate both)
125+
run: |
126+
dxt-explorer --debug --start 3.7 --end 3.9 --from 1 --to 100 sample/jeanbez_8_benchmark_parallel_id45195555_8-6-50011-11681279140261054765_1628283217.darshan
127+
128+
- name: Upload Artifact
129+
if: always()
130+
uses: actions/upload-artifact@v4
131+
with:
132+
name: tests-${{ matrix.os }}-${{ matrix.python-version }}
133+
path: sample/**
134+
retention-days: 1
135+
136+

docs/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ sphinxcontrib-jsmath==1.0.1
88
sphinxcontrib-qthelp==2.0.0
99
sphinxcontrib-serializinghtml==2.0.0
1010
sphinxemoji==0.3.1
11-
urllib3==2.5.0
11+
urllib3==2.6.3

explorer/dxt.py

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@
3030
import subprocess
3131
import webbrowser
3232
import pandas as pd
33-
import pkg_resources
3433
import pyranges as pr
3534
import logging.handlers
3635
import pyarrow.feather as feather
3736
# import darshan.backend.cffi_backend as darshanll
3837

38+
from importlib import resources
3939
from explorer import version as dxt_version
4040
from packaging import version
4141

@@ -298,7 +298,7 @@ def create_dataframe(
298298
temp_result["end"] = temp_result["end"].round(decimals=4)
299299

300300
temp_result.index.name = "segment"
301-
temp_result.reset_index(inplace=True)
301+
temp_result = temp_result.reset_index()
302302
temp_result = temp_result.reindex(columns=column_names)
303303

304304
df.append(temp_result)
@@ -327,7 +327,7 @@ def create_dataframe(
327327
temp_result["end"] = temp_result["end"].round(decimals=4)
328328

329329
temp_result.index.name = "segment"
330-
temp_result.reset_index(inplace=True)
330+
temp_result = temp_result.reset_index()
331331
temp_result = temp_result.reindex(columns=column_names)
332332

333333
df.append(temp_result)
@@ -422,8 +422,8 @@ def merge_overlapping_io_phases(self, overlapping_df, df, module):
422422
]
423423
)
424424

425-
overlapping_df_end = overlapping_df[["End"]].to_numpy()
426-
overlapping_df_start = overlapping_df[["Start"]].to_numpy()
425+
overlapping_df_end = overlapping_df["End"].to_numpy()
426+
overlapping_df_start = overlapping_df["Start"].to_numpy()
427427
interval_duration = 0
428428

429429
for i in range(len(overlapping_df_end) - 1):
@@ -513,7 +513,8 @@ def merge_overlapping_io_phases(self, overlapping_df, df, module):
513513
threshold,
514514
]
515515

516-
io_phases_df.dropna(inplace=True)
516+
io_phases_df = io_phases_df.dropna()
517+
517518
return io_phases_df
518519

519520
def calculate_io_phases(
@@ -730,7 +731,7 @@ def generate_plot(self, file, report):
730731
self.prefix, file_id, "snapshot", snapshot, "operation"
731732
)
732733
path = "plots/operation.py"
733-
script = pkg_resources.resource_filename(__name__, path)
734+
script = str(resources.files(__package__ or __name__).joinpath(path))
734735

735736
command = "python3 {} -f {}.{}.{}-{}.dxt -i {}.{}.{}-{}.io_phases {} {} -o {} -x {} -t {} -r {}".format(
736737
script,
@@ -792,7 +793,7 @@ def generate_plot(self, file, report):
792793
self.prefix, file_id, "operation"
793794
)
794795
path = "plots/operation.py"
795-
script = pkg_resources.resource_filename(__name__, path)
796+
script = str(resources.files(__package__ or __name__).joinpath(path))
796797

797798
command = "python3 {} -f {}.{}.dxt -i {}.{}.io_phases{} {} -o {} -x {}".format(
798799
script,
@@ -864,7 +865,7 @@ def generate_transfer_plot(self, file, report):
864865
output_file = "{}/{}-{}.html".format(self.prefix, file_id, "transfer")
865866

866867
path = "plots/transfer.py"
867-
script = pkg_resources.resource_filename(__name__, path)
868+
script = str(resources.files(__package__ or __name__).joinpath(path))
868869

869870
command = "python3 {} -f {}.{}.dxt {} -o {} -x {}".format(
870871
script, file, file_id, limits, output_file, file_name
@@ -912,7 +913,7 @@ def generate_spatiality_plot(self, file, report):
912913
output_file = "{}/{}-{}.html".format(self.prefix, file_id, "spatiality")
913914

914915
path = "plots/spatiality.py"
915-
script = pkg_resources.resource_filename(__name__, path)
916+
script = str(resources.files(__package__ or __name__).joinpath(path))
916917

917918
command = "python3 {} -f {}.{}.dxt -o {} -x {}".format(
918919
script, file, file_id, output_file, file_name
@@ -959,7 +960,7 @@ def generate_phase_plot(self, file, report):
959960
for file_id, file_name in file_ids.items():
960961
output_file = "{}/{}-{}.html".format(self.prefix, file_id, "io_phase")
961962
path = "plots/io_phase.py"
962-
script = pkg_resources.resource_filename(__name__, path)
963+
script = str(resources.files(__package__ or __name__).joinpath(path))
963964

964965
command = "python3 {} -f {}.{}.io_phases -o {} -x {}".format(
965966
script, file, file_id, output_file, file_name
@@ -1007,7 +1008,7 @@ def generate_ost_usage_operation_plot(self, file, report):
10071008
self.prefix, file_id, "ost_usage_operation"
10081009
)
10091010
path = "plots/ost_usage_operation.py"
1010-
script = pkg_resources.resource_filename(__name__, path)
1011+
script = str(resources.files(__package__ or __name__).joinpath(path))
10111012

10121013
command = "python3 {} -f {}.{}.dxt -o {} -x {}".format(
10131014
script, file, file_id, output_file, file_name
@@ -1057,7 +1058,7 @@ def generate_ost_usage_transfer_plot(self, file, report):
10571058
self.prefix, file_id, "ost_usage_transfer"
10581059
)
10591060
path = "plots/ost_usage_transfer.py"
1060-
script = pkg_resources.resource_filename(__name__, path)
1061+
script = str(resources.files(__package__ or __name__).joinpath(path))
10611062

10621063
command = "python3 {} -f {}.{}.dxt -o {} -x {}".format(
10631064
script, file, file_id, output_file, file_name

0 commit comments

Comments
 (0)