Skip to content

Commit 80a7f20

Browse files
Alberto-oAlberto-o
authored
Merge pull request #127 from OpenSEMBA/debugging-movies
Debugging MPI problem with movies and mapvtk
2 parents e0a95c8 + b333556 commit 80a7f20

File tree

9 files changed

+7261
-75
lines changed

9 files changed

+7261
-75
lines changed

‎.github/workflows/ubuntu.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ jobs:
8888

8989
- name: Run python tests
9090
env:
91+
SEMBA_FDTD_ENABLE_MPI: ${{ matrix.mpi }}
9192
SEMBA_FDTD_ENABLE_MTLN: ${{ matrix.mtln }}
9293
SEMBA_FDTD_ENABLE_HDF: ${{ matrix.hdf }}
9394
run: python -m pytest test/

‎.github/workflows/windows.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ jobs:
6969

7070
- name: Run python tests (except codemodel)
7171
env:
72+
SEMBA_FDTD_ENABLE_MPI: ${{ matrix.mpi }}
7273
SEMBA_FDTD_ENABLE_MTLN: ${{ matrix.mtln }}
7374
SEMBA_FDTD_ENABLE_HDF: ${{ matrix.hdf }}
7475
run: python -m pytest -m 'not codemodel' test/

‎src_main_pub/timestepping.F90‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,44 @@
1+
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
2+
! MIT License
3+
!
4+
! Copyright (c) 2023 University of Granada
5+
!
6+
! Permission is hereby granted, free of charge, to any person obtaining a copy
7+
! of this software and associated documentation files (the "Software"), to deal
8+
! in the Software without restriction, including without limitation the rights
9+
! to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
! copies of the Software, and to permit persons to whom the Software is
11+
! furnished to do so, subject to the following conditions:
12+
!
13+
! The above copyright notice and this permission notice shall be included in all
14+
! copies or substantial portions of the Software.
15+
!
16+
! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
! AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
! LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
! OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
! SOFTWARE.
23+
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
24+
25+
26+
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
27+
! SEMBA_FDTD sOLVER MODULE
28+
! Creation date Date : April, 8, 2010
29+
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
30+
31+
!__________________________________________________________________________________________________
32+
!******************************** REVISAR PARA PGI (CRAY) *****************************************
33+
!---> AdvanceMultiportE
34+
!---> AdvanceAnisMultiportE
35+
!---> AdvanceMultiportH
36+
!---> AdvanceAnisMultiportH
37+
!---> dfUpdateE
38+
!---> dfUpdateH
39+
!---> MinusCloneMagneticPMC
40+
!________________________________________________________________________________________
41+
142
module Solver
243

344
use fdetypes
@@ -2782,8 +2823,11 @@ subroutine fillMtag(sgg,sggMiEx, sggMiEy, sggMiEz, sggMiHx, sggMiHy, sggMiHz,sgg
27822823
integer(kind = 4) :: i, j, k
27832824
integer(kind = INTEGERSIZEOFMEDIAMATRICES) :: medio1,medio2,medio3,medio4,medio5
27842825
logical :: mediois1,mediois2,mediois3,mediois4
2785-
2786-
2826+
integer, dimension(3) :: lbx, lby, lbz
2827+
lbx = lbound(tag_numbers%face%x)
2828+
lby = lbound(tag_numbers%face%y)
2829+
lbz = lbound(tag_numbers%face%z)
2830+
27872831
mediois3=.true.; mediois4=.true.
27882832
#ifdef CompileWithOpenMP
27892833
!$OMP PARALLEL DO DEFAULT(SHARED) private (i,j,k,medio1,medio2,medio3,medio4,medio5,mediois1,mediois2,mediois3,mediois4)
@@ -2801,7 +2845,7 @@ subroutine fillMtag(sgg,sggMiEx, sggMiEy, sggMiEz, sggMiHx, sggMiHy, sggMiHz,sgg
28012845
mediois3= .true. !.not.((medio5==1).and.(((sggMiHx(i-1,j,k)/=1).or.(sggMiHx(i+1,j,k)/=1)))) !esta condicion en realidad no detecta alabeos de una celda que siendo slots son acoples de un agujerito solo en el peor de los casos
28022846
if ((mediois1.or.mediois2).and.(mediois3)) then
28032847
!solo lo hace con celdas de vacio porque en particular el mismo medio sgbc con diferentes orientaciones tiene distintos indices de medio y lo activaria erroneamente si lo hago para todos los medios
2804-
tag_numbers%face%x(i,j,k)=-ibset(iabs(tag_numbers%face%x(i,j,k)),3)
2848+
tag_numbers%face%x(i+lbx(1)-1,j+lbx(2)-1,k+lbx(3)-1)=-ibset(iabs(tag_numbers%face%x(i+lbx(1)-1,j+lbx(2)-1,k+lbx(3)-1)),3)
28052849
!ojo no cambiar: interacciona con observation tags 141020 !151020 a efectos de mapvtk el signo importa
28062850
endif
28072851
End do
@@ -2823,7 +2867,7 @@ subroutine fillMtag(sgg,sggMiEx, sggMiEy, sggMiEz, sggMiHx, sggMiHy, sggMiHz,sgg
28232867
mediois2= (medio5==1).and.(medio3/=1).and.(medio4/=1).and.(medio1==1).and.(medio2==1)
28242868
mediois3= .true. !.not.((medio5==1).and.(((sggMiHy(i,j-1,k)/=1).or.(sggMiHy(i,j+1,k)/=1))))
28252869
if ((mediois1.or.mediois2).and.(mediois3)) then
2826-
tag_numbers%face%y(i,j,k)=-ibset(iabs(tag_numbers%face%y(i,j,k)),4)
2870+
tag_numbers%face%y(i+lby(1)-1,j+lby(2)-1,k+lby(3)-1)=-ibset(iabs(tag_numbers%face%y(i+lby(1)-1,j+lby(2)-1,k+lby(3)-1)),4)
28272871
endif
28282872
End do
28292873
End do
@@ -2844,7 +2888,7 @@ subroutine fillMtag(sgg,sggMiEx, sggMiEy, sggMiEz, sggMiHx, sggMiHy, sggMiHz,sgg
28442888
mediois2= (medio5==1).and.(medio3/=1).and.(medio4/=1).and.(medio1==1).and.(medio2==1)
28452889
mediois3= .true. !.not.((medio5==1).and.(((sggMiHz(i,j,k-1)/=1).or.(sggMiHz(i,j,k+1)/=1))))
28462890
if ((mediois1.or.mediois2).and.(mediois3)) then
2847-
tag_numbers%face%z(i,j,k)=-ibset(iabs(tag_numbers%face%z(i,j,k)),5)
2891+
tag_numbers%face%z(i+lbz(1)-1,j+lbz(2)-1,k+lbz(3)-1)=-ibset(iabs(tag_numbers%face%z(i+lbz(1)-1,j+lbz(2)-1,k+lbz(3)-1)),5)
28482892
endif
28492893
End do
28502894
End do

‎src_pyWrapper/pyWrapper.py‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ class Probe():
2424
+ BULK_CURRENT_PROBE_TAGS \
2525
+ POINT_PROBE_TAGS \
2626
+ FAR_FIELD_TAG \
27-
+ MOVIE_TAGS
27+
+ MOVIE_TAGS
28+
2829
def __init__(self, probe_filename):
2930
if isinstance(probe_filename, os.PathLike):
3031
self.filename = probe_filename.as_posix()
@@ -92,7 +93,7 @@ def __init__(self, probe_filename):
9293
self.field, self.direction = Probe._getFieldAndDirection(tag)
9394
self.cell_init, self.cell_end = \
9495
Probe._positionStrToTwoCells(position_str)
95-
96+
9697
if self.domainType == 'time':
9798
self.data = self.data.rename(columns={
9899
't': 'time',
@@ -179,17 +180,32 @@ def _getFieldAndDirection(tag: str):
179180

180181

181182
class FDTD():
182-
def __init__(self, input_filename, path_to_exe=None, flags=[], run_in_folder=None):
183+
def __init__(self, input_filename, path_to_exe=None,
184+
flags=None, run_in_folder=None, mpi_command=None):
185+
183186
self._setFilename(input_filename)
184187

185188
if path_to_exe is None:
186-
self.path_to_exe = os.path.join(
187-
os.getcwd(), DEFAULT_SEMBA_FDTD_PATH)
189+
semba_exe = \
190+
os.path.join(os.getcwd(), DEFAULT_SEMBA_FDTD_PATH)
191+
else:
192+
semba_exe = path_to_exe
193+
assert os.path.isfile(semba_exe)
194+
195+
if mpi_command is None:
196+
mpi_command_parts = []
188197
else:
189-
self.path_to_exe = path_to_exe
190-
assert os.path.isfile(self.path_to_exe)
198+
mpi_command_parts = mpi_command.split()
199+
200+
if flags is None:
201+
flags = []
202+
elif isinstance(flags, str):
203+
flags = flags.split()
204+
205+
case_name = self.getCaseName() + ".json"
206+
self.run_command = \
207+
mpi_command_parts + [semba_exe] + ["-i", case_name] + flags
191208

192-
self.flags = flags
193209
self._hasRun = False
194210

195211
if run_in_folder != None:
@@ -263,9 +279,7 @@ def run(self):
263279
json.dump(self._input, open(self._filename, 'w'))
264280

265281
os.chdir(self.getFolder())
266-
case_name = self.getCaseName() + ".json"
267-
self.output = subprocess.run(
268-
[self.path_to_exe, "-i", case_name]+self.flags)
282+
self.output = subprocess.run(self.run_command)
269283

270284
self._hasRun = True
271285
assert self.hasFinishedSuccessfully()

‎test/pyWrapper/test_integration.py‎

Lines changed: 93 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,19 @@ def test_towel_hanger_case_creates_output_probes(tmp_path):
3939
assert countLinesInFile(probe_mid[0]) == 3
4040
assert countLinesInFile(probe_end[0]) == 3
4141

42+
@no_mpi_skip
43+
def test_airplane_case_with_mpi(tmp_path):
44+
fn = CASES_FOLDER + 'airplane/airplane.fdtd.json'
45+
solver = FDTD(fn,
46+
path_to_exe=SEMBA_EXE,
47+
run_in_folder=tmp_path,
48+
flags=['-mapvtk'],
49+
mpi_command='mpirun -np 2')
50+
solver.run()
51+
52+
vtkmapfile = solver.getVTKMap()
53+
assert os.path.isfile(vtkmapfile)
54+
4255

4356
def test_sphere_case_with_far_field_probe_launches(tmp_path):
4457
fn = CASES_FOLDER + 'sphere/sphere.fdtd.json'
@@ -66,29 +79,34 @@ def test_tagnumbers_3_surfaces(tmp_path):
6679
solver['general']['numberOfSteps'] = 1
6780

6881
solver.run()
69-
82+
7083
vtkmapfile = solver.getVTKMap()
7184
assert os.path.isfile(vtkmapfile)
7285

73-
face_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'tagnumber')
86+
face_tag_dict = createPropertyDictionary(
87+
vtkmapfile, celltype=9, property='tagnumber')
7488
assert face_tag_dict[64] == 4
7589
assert face_tag_dict[128] == 4
7690
assert face_tag_dict[192] == 4
7791

78-
line_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'tagnumber')
92+
line_tag_dict = createPropertyDictionary(
93+
vtkmapfile, celltype=3, property='tagnumber')
7994
assert line_tag_dict[64] == 8
8095
assert line_tag_dict[128] == 4
8196
assert line_tag_dict[192] == 4
82-
83-
face_media_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'mediatype')
84-
assert face_media_dict[0] == 4 #PEC surface
85-
assert face_media_dict[304] == 4 #SGBC surface
86-
assert face_media_dict[305] == 4 #SGBC surface
87-
88-
line_media_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'mediatype')
89-
assert line_media_dict[0.5] == 8 #PEC line
90-
assert line_media_dict[3.5] == 8 #SGBC line
91-
97+
98+
face_media_dict = createPropertyDictionary(
99+
vtkmapfile, celltype=9, property='mediatype')
100+
assert face_media_dict[0] == 4 # PEC surface
101+
assert face_media_dict[304] == 4 # SGBC surface
102+
assert face_media_dict[305] == 4 # SGBC surface
103+
104+
line_media_dict = createPropertyDictionary(
105+
vtkmapfile, celltype=3, property='mediatype')
106+
assert line_media_dict[0.5] == 8 # PEC line
107+
assert line_media_dict[3.5] == 8 # SGBC line
108+
109+
92110
def test_tagnumbers_1_volume(tmp_path):
93111
fn = CASES_FOLDER + 'tagNumber_mediaType/pec_volume.fdtd.json'
94112
solver = FDTD(input_filename=fn, path_to_exe=SEMBA_EXE,
@@ -100,18 +118,23 @@ def test_tagnumbers_1_volume(tmp_path):
100118
vtkmapfile = solver.getVTKMap()
101119
assert os.path.isfile(vtkmapfile)
102120

103-
face_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'tagnumber')
121+
face_tag_dict = createPropertyDictionary(
122+
vtkmapfile, celltype=9, property='tagnumber')
104123
assert face_tag_dict[64] == 36
105-
106-
line_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'tagnumber')
124+
125+
line_tag_dict = createPropertyDictionary(
126+
vtkmapfile, celltype=3, property='tagnumber')
107127
assert len(line_tag_dict) == 0
108-
109-
face_media_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'mediatype')
110-
assert face_media_dict[0] == 36 #PEC surface
111-
112-
line_media_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'mediatype')
128+
129+
face_media_dict = createPropertyDictionary(
130+
vtkmapfile, celltype=9, property='mediatype')
131+
assert face_media_dict[0] == 36 # PEC surface
132+
133+
line_media_dict = createPropertyDictionary(
134+
vtkmapfile, celltype=3, property='mediatype')
113135
assert len(line_media_dict) == 0
114136

137+
115138
def test_tagnumbers_2_volumes(tmp_path):
116139
fn = CASES_FOLDER + 'tagNumber_mediaType/pec_volumes.fdtd.json'
117140
solver = FDTD(input_filename=fn, path_to_exe=SEMBA_EXE,
@@ -123,19 +146,24 @@ def test_tagnumbers_2_volumes(tmp_path):
123146
vtkmapfile = solver.getVTKMap()
124147
assert os.path.isfile(vtkmapfile)
125148

126-
face_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'tagnumber')
149+
face_tag_dict = createPropertyDictionary(
150+
vtkmapfile, celltype=9, property='tagnumber')
127151
assert face_tag_dict[64] == 36
128152
assert face_tag_dict[128] == 36
129-
130-
line_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'tagnumber')
153+
154+
line_tag_dict = createPropertyDictionary(
155+
vtkmapfile, celltype=3, property='tagnumber')
131156
assert len(line_tag_dict) == 0
132-
133-
face_media_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'mediatype')
134-
assert face_media_dict[0] == 72 #PEC surface
135-
136-
line_media_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'mediatype')
157+
158+
face_media_dict = createPropertyDictionary(
159+
vtkmapfile, celltype=9, property='mediatype')
160+
assert face_media_dict[0] == 72 # PEC surface
161+
162+
line_media_dict = createPropertyDictionary(
163+
vtkmapfile, celltype=3, property='mediatype')
137164
assert len(line_media_dict) == 0
138165

166+
139167
def test_tagnumbers_1_line(tmp_path):
140168
fn = CASES_FOLDER + 'tagNumber_mediaType/pec_line.fdtd.json'
141169
solver = FDTD(input_filename=fn, path_to_exe=SEMBA_EXE,
@@ -147,18 +175,23 @@ def test_tagnumbers_1_line(tmp_path):
147175
vtkmapfile = solver.getVTKMap()
148176
assert os.path.isfile(vtkmapfile)
149177

150-
face_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'tagnumber')
178+
face_tag_dict = createPropertyDictionary(
179+
vtkmapfile, celltype=9, property='tagnumber')
151180
assert len(face_tag_dict) == 0
152-
153-
line_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'tagnumber')
181+
182+
line_tag_dict = createPropertyDictionary(
183+
vtkmapfile, celltype=3, property='tagnumber')
154184
assert line_tag_dict[64] == 2
155-
156-
face_media_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'mediatype')
185+
186+
face_media_dict = createPropertyDictionary(
187+
vtkmapfile, celltype=9, property='mediatype')
157188
assert len(face_media_dict) == 0
158-
159-
line_media_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'mediatype')
160-
assert line_media_dict[0.5] == 2 #PEC line
161-
189+
190+
line_media_dict = createPropertyDictionary(
191+
vtkmapfile, celltype=3, property='mediatype')
192+
assert line_media_dict[0.5] == 2 # PEC line
193+
194+
162195
def test_tagnumbers_volume_and_surfaces(tmp_path):
163196
fn = CASES_FOLDER + 'tagNumber_mediaType/volume_and_surfaces.fdtd.json'
164197
solver = FDTD(input_filename=fn, path_to_exe=SEMBA_EXE,
@@ -170,26 +203,31 @@ def test_tagnumbers_volume_and_surfaces(tmp_path):
170203
vtkmapfile = solver.getVTKMap()
171204
assert os.path.isfile(vtkmapfile)
172205

173-
face_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'tagnumber')
206+
face_tag_dict = createPropertyDictionary(
207+
vtkmapfile, celltype=9, property='tagnumber')
174208
assert face_tag_dict[64] == 6
175209
assert face_tag_dict[128] == 1
176210
assert face_tag_dict[192] == 1
177-
178-
line_tag_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'tagnumber')
211+
212+
line_tag_dict = createPropertyDictionary(
213+
vtkmapfile, celltype=3, property='tagnumber')
179214
assert line_tag_dict[64] == 1
180215
assert line_tag_dict[128] == 4
181216
assert line_tag_dict[192] == 3
182-
183-
face_media_dict = createPropertyDictionary(vtkmapfile, celltype = 9, property = 'mediatype')
184-
assert face_media_dict[-1] == 1 #PEC surface
185-
assert face_media_dict[0] == 6 #PEC surface
186-
assert face_media_dict[305] == 1 #SGBC surface
187-
188-
line_media_dict = createPropertyDictionary(vtkmapfile, celltype = 3, property = 'mediatype')
189-
assert line_media_dict[-0.5] == 4 #PMC line
190-
assert line_media_dict[0.5] == 1 #PEC line
191-
assert line_media_dict[3.5] == 3 #SGBC line
192-
217+
218+
face_media_dict = createPropertyDictionary(
219+
vtkmapfile, celltype=9, property='mediatype')
220+
assert face_media_dict[-1] == 1 # PEC surface
221+
assert face_media_dict[0] == 6 # PEC surface
222+
assert face_media_dict[305] == 1 # SGBC surface
223+
224+
line_media_dict = createPropertyDictionary(
225+
vtkmapfile, celltype=3, property='mediatype')
226+
assert line_media_dict[-0.5] == 4 # PMC line
227+
assert line_media_dict[0.5] == 1 # PEC line
228+
assert line_media_dict[3.5] == 3 # SGBC line
229+
230+
193231
def test_tagnumbers_count_bug(tmp_path):
194232
fn = CASES_FOLDER + 'tagNumber_mediaType/count_bug.fdtd.json'
195233
solver = FDTD(input_filename=fn, path_to_exe=SEMBA_EXE,
@@ -201,10 +239,10 @@ def test_tagnumbers_count_bug(tmp_path):
201239
solver["materialAssociations"][1]["materialId"] = 1
202240
solver["materialAssociations"][2]["materialId"] = 3
203241
solver.cleanUp()
204-
solver.run()
205-
242+
solver.run()
243+
206244
solver["materialAssociations"][0]["materialId"] = 3
207245
solver["materialAssociations"][1]["materialId"] = 3
208246
solver["materialAssociations"][2]["materialId"] = 1
209247
solver.cleanUp()
210-
solver.run()
248+
solver.run()

0 commit comments

Comments
 (0)