Compilation Error for Fake Multi-GPU Devices

[avik-pal]

diff --git a/sharding/sharded_baroclinic_instability_simulation_run.jl b/sharding/sharded_baroclinic_instability_simulation_run.jl
index 147ee0f..7fda52b 100644
--- a/sharding/sharded_baroclinic_instability_simulation_run.jl
+++ b/sharding/sharded_baroclinic_instability_simulation_run.jl
@@ -29,8 +29,10 @@ Reactant.Compiler.WHILE_CONCAT[] = true
 # Reactant.Compiler.SUM_TO_REDUCEWINDOW[] = true
 # Reactant.Compiler.AGGRESSIVE_SUM_TO_CONV[] = true
 
-GordonBell25.initialize(; single_gpu_per_process=false)
-@show Ndev = length(Reactant.devices())
+# GordonBell25.initialize(; single_gpu_per_process=false)
+devs = repeat([Reactant.devices()[1]], 8)
+# @show Ndev = length(Reactant.devices())
+@show Ndev = length(devs)
 
 Rx, Ry = factors(Ndev)
 if Ndev == 1
@@ -39,7 +41,8 @@ if Ndev == 1
 else
     arch = Oceananigans.Distributed(
         Oceananigans.ReactantState();
-        partition = Partition(Rx, Ry, 1)
+        partition = Partition(Rx, Ry, 1),
+       devices=devs
     )
     rank = Reactant.Distributed.local_rank()
 end

loc(callsite(fused<#llvm.di_subprogram<id = distinct[0]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"float.jl" in ".">, name = "*;", linkageName = "*", file = <"float.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["float.jl":493:0] at callsite(fused<#llvm.di_subprogram<id = distinct[2]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"operators.jl" in ".">, name = "*;", linkageName = "*", file = <"operators.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["operators.jl":596:0] at callsite(fused<#llvm.di_subprogram<id = distinct[3]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/upwind_biased_advective_fluxes.jl" in ".">, name = "advective_tracer_flux_y;", linkageName = "advective_tracer_flux_y", file = <"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/upwind_biased_advective_fluxes.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/upwind_biased_advective_fluxes.jl":112:0] at callsite(fused<#llvm.di_subprogram<id = distinct[4]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/tracer_advection_operators.jl" in ".">, name = "_advective_tracer_flux_y;", linkageName = "_advective_tracer_flux_y", file = <"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/tracer_advection_operators.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/tracer_advection_operators.jl":3:0] at callsite(fused<#llvm.di_subprogram<id = distinct[5]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Operators/difference_operators.jl" in ".">, name = "\CE\B4y\E1\B5\83\E1\B6\9C\E1\B5\83;", linkageName = "\CE\B4y\E1\B5\83\E1\B6\9C\E1\B5\83", file = <"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Operators/difference_operators.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Operators/difference_operators.jl":23:0] at callsite(fused<#llvm.di_subprogram<id = distinct[6]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/tracer_advection_operators.jl" in ".">, name = "div_Uc;", linkageName = "div_Uc", file = <"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/tracer_advection_operators.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Advection/tracer_advection_operators.jl":31:0] at callsite(fused<#llvm.di_subprogram<id = distinct[7]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl" in ".">, name = "hydrostatic_free_surface_tracer_tendency;", linkageName = "hydrostatic_free_surface_tracer_tendency", file = <"/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["/home/avik-pal/.julia/packages/Oceananigans/CGCJ5/src/Models/HydrostaticFreeSurfaceModels/hydrostatic_free_surface_tendency_kernel_functions.jl":137:0] at callsite(fused<#llvm.di_subprogram<id = distinct[8]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, scope = #llvm.di_file<"/home/avik-pal/.julia/packages/KernelAbstractions/lGrz7/src/macros.jl" in ".">, name = "gpu_compute_hydrostatic_free_surface_Gc!;", linkageName = "gpu_compute_hydrostatic_free_surface_Gc!", file = <"/home/avik-pal/.julia/packages/KernelAbstractions/lGrz7/src/macros.jl" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["/home/avik-pal/.julia/packages/KernelAbstractions/lGrz7/src/macros.jl":324:0] at fused<#llvm.di_subprogram<id = distinct[9]<>, compileUnit = <id = distinct[1]<>, sourceLanguage = DW_LANG_Julia, file = <"julia" in ".">, producer = "julia", isOptimized = true, emissionKind = None, nameTableKind = None>, name = "gpu_compute_hydrostatic_free_surface_Gc!", linkageName = "julia_gpu_compute_hydrostatic_free_surface_Gc!_74620", file = <"none" in ".">, subprogramFlags = "Definition|Optimized", type = <>>>["none":0:0]))))))))): error: 'stablehlo.multiply' op requires compatible types for all operands and results

@wsmoses any idea why this happens? I would have expected this to fail but not here

[wsmoses]

does it work otherwise [just to confirm we didnt trigger some different pass failing]

[avik-pal]

Single GPU goes through

[wsmoses]

does real multi gpu work, but either way still weird -- do you have the module and passes which err?