Skip to content

TPU error #107

@wsmoses

Description

@wsmoses
julia> include("ocean-climate-simulation/ocean_climate_simulation_compile.jl")
┌ Warning: You are using Julia v1.11 or later!"
│ Oceananigans is currently tested on Julia v1.10."
│ If you find issues with Julia v1.11 or later,"
│ please report at https://github.com/CliMA/Oceananigans.jl/issues/new
└ @ Oceananigans ~/.julia/packages/Oceananigans/OfIsz/src/Oceananigans.jl:118
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
I0000 00:00:1743047145.229194  638303 pjrt_api.cc:99] GetPjrtApi was found for tpu at /home/wmoses/.julia/scratchspaces/3c362404-f566-11ee-1572-e11a4b42c853/libtpu/libtpu.so
I0000 00:00:1743047145.231099  638303 pjrt_api.cc:78] PJRT_Api is set for device type tpu
I0000 00:00:1743047145.231120  638303 pjrt_api.cc:145] The PJRT plugin has PJRT API version 0.67. The framework PJRT API version is 0.67.
2025-03-27 03:45:51.908560: I external/xla/xla/pjrt/pjrt_c_api_client.cc:130] PjRtCApiClient created.
[ Info: Generating model...
┌ Error: Compilation failed, MLIR module written to /tmp/reactant_PUFjxH/jl_VuDMFUxyFa.mlir
└ @ Reactant.MLIR.IR ~/Reactant.jl/src/mlir/IR/Pass.jl:93
ERROR: LoadError: Scalar indexing is disallowed.
Invocation of getindex(::ConcretePJRTArray, ::Vararg{Int, N}) resulted in scalar indexing of a GPU array.
This is typically caused by calling an iterating implementation of a method.
Such implementations *do not* execute on the GPU, but very slowly on the CPU,
and therefore should be avoided.

If you want to allow scalar iteration, use `allowscalar` or `@allowscalar`
to enable scalar iteration globally or for the operations in question.
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:35
  [2] errorscalar(op::String)
    @ GPUArraysCore ~/.julia/packages/GPUArraysCore/aNaXo/src/GPUArraysCore.jl:151
  [3] _assertscalar(op::String, behavior::GPUArraysCore.ScalarIndexing)
    @ GPUArraysCore ~/.julia/packages/GPUArraysCore/aNaXo/src/GPUArraysCore.jl:124
  [4] assertscalar(op::String)
    @ GPUArraysCore ~/.julia/packages/GPUArraysCore/aNaXo/src/GPUArraysCore.jl:112
  [5] getindex(::ConcretePJRTArray{Float64, 3, 1, Reactant.Sharding.ShardInfo{Reactant.Sharding.NoSharding, Nothing}}, ::Int64, ::Int64, ::Int64)
    @ Reactant ~/Reactant.jl/src/ConcreteRArray.jl:259
  [6] getindex
    @ ./subarray.jl:320 [inlined]
  [7] _getindex
    @ ./abstractarray.jl:1358 [inlined]
  [8] getindex
    @ ./abstractarray.jl:1312 [inlined]
  [9] copyto_unaliased!(deststyle::IndexCartesian, dest::SubArray{Float64, 3, ConcretePJRTArray{…}, Tuple{…}, false}, srcstyle::IndexCartesian, src::SubArray{Float64, 3, ConcretePJRTArray{…}, Tuple{…}, false})
    @ Base ./abstractarray.jl:1095
 [10] copyto!
    @ ./abstractarray.jl:1061 [inlined]
 [11] copyto!
    @ ./broadcast.jl:966 [inlined]
 [12] copyto!
    @ ./broadcast.jl:925 [inlined]
 [13] materialize!
    @ ./broadcast.jl:883 [inlined]
 [14] materialize!(dest::SubArray{Float64, 3, ConcretePJRTArray{…}, Tuple{…}, false}, bc::Base.Broadcast.Broadcasted{Base.Broadcast.DefaultArrayStyle{…}, Nothing, typeof(identity), Tuple{…}})
    @ Base.Broadcast ./broadcast.jl:880
 [15] set_to_field!(u::Field{…}, v::Field{…})
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:144
 [16] set!(u::Field{…}, v::Field{…})
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:37
 [17] set_to_function!(u::Field{Center, Center, Nothing, Nothing, OrthogonalSphericalShellGrid{…}, Tuple{…}, OffsetArrays.OffsetArray{…}, Float64, FieldBoundaryConditions{…}, Nothing, Oceananigans.Fields.FieldBoundaryBuffers{…}}, f::Function)
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:95
 [18] set!(u::Field{Center, Center, Nothing, Nothing, OrthogonalSphericalShellGrid{…}, Tuple{…}, OffsetArrays.OffsetArray{…}, Float64, FieldBoundaryConditions{…}, Nothing, Oceananigans.Fields.FieldBoundaryBuffers{…}}, f::Function)
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:35
 [19] materialize_immersed_boundary(grid::OrthogonalSphericalShellGrid{…}, ib::GridFittedBottom{…})
    @ OceananigansReactantExt.Grids ~/.julia/packages/Oceananigans/OfIsz/ext/OceananigansReactantExt/Grids/serial_grids.jl:15
 [20] ImmersedBoundaryGrid(grid::OrthogonalSphericalShellGrid{…}, ib::GridFittedBottom{…}; active_cells_map::Bool, active_z_columns::Bool)
    @ Oceananigans.ImmersedBoundaries ~/.julia/packages/Oceananigans/OfIsz/src/ImmersedBoundaries/immersed_boundary_grid.jl:47
 [21] macro expansion
    @ ~/.julia/juliaup/julia-1.11.4+0.x64.linux.gnu/share/julia/stdlib/v1.11/Profile/src/Profile.jl:59 [inlined]
 [22] gaussian_islands_tripolar_grid(arch::ReactantState, resolution::Int64, Nz::Int64)
    @ GordonBell25 ~/GB-25/src/data_free_ocean_climate_model.jl:22
 [23] data_free_ocean_climate_model_init(arch::ReactantState; resolution::Int64, Nz::Int64)
    @ GordonBell25 ~/GB-25/src/data_free_ocean_climate_model.jl:110
 [24] data_free_ocean_climate_model_init(arch::ReactantState)
    @ GordonBell25 ~/GB-25/src/data_free_ocean_climate_model.jl:102
 [25] top-level scope
    @ ~/GB-25/ocean-climate-simulation/ocean_climate_simulation_compile.jl:11
 [26] include(fname::String)
    @ Main ./sysimg.jl:38
 [27] top-level scope
    @ REPL[1]:1
in expression starting at /home/wmoses/GB-25/ocean-climate-simulation/ocean_climate_simulation_compile.jl:11

caused by: module @reactant_Base.Br... attributes {mhlo.num_partitions = 1 : i64, mhlo.num_replicas = 1 : i64} {
  func.func @main(%arg0: tensor<1x104x194xf64> {reactant.donated}) {
    return
  }
}
INVALID_ARGUMENT: Failed to serialize StableHLO;

Detailed error from MLIR: <unknown>:0: error: failed to legalize operation 'func.func' that was explicitly marked illegal
<unknown>:0: note: see current operation: 
"func.func"() <{arg_attrs = [{reactant.donated}], function_type = (tensor<1x104x194xf64>) -> (), sym_name = "main"}> ({
^bb0(%arg0: tensor<1x104x194xf64>):
  "func.return"() : () -> ()
}) : () -> ()


Stacktrace:
  [1] reactant_err(msg::Cstring)
    @ Reactant.XLA ~/Reactant.jl/src/xla/Utils.jl:12
  [2] (::Reactant.XLA.PJRT.var"#19#20"{Bool, Vector{Int64}, Bool, Reactant.XLA.PJRT.Client, Reactant.MLIR.IR.Module, Int64})()
    @ Reactant.XLA.PJRT ~/Reactant.jl/src/xla/PJRT/LoadedExecutable.jl:83
  [3] try_compile_dump_mlir(f::Reactant.XLA.PJRT.var"#19#20"{Bool, Vector{Int64}, Bool, Reactant.XLA.PJRT.Client, Reactant.MLIR.IR.Module, Int64}, mod::Reactant.MLIR.IR.Module, pm::Nothing)
    @ Reactant.MLIR.IR ~/Reactant.jl/src/mlir/IR/Pass.jl:101
  [4] try_compile_dump_mlir
    @ ~/Reactant.jl/src/mlir/IR/Pass.jl:100 [inlined]
  [5] #compile#18
    @ ~/Reactant.jl/src/xla/PJRT/LoadedExecutable.jl:82 [inlined]
  [6] compile_xla(f::Function, args::Tuple{ConcretePJRTArray{Float64, 3, 1, Reactant.Sharding.ShardInfo{Reactant.Sharding.NoSharding, Nothing}}}; client::Nothing, kwargs::@Kwargs{})
    @ Reactant.Compiler ~/Reactant.jl/src/Compiler.jl:2006
  [7] compile_xla
    @ ~/Reactant.jl/src/Compiler.jl:1963 [inlined]
  [8] compile(f::Function, args::Tuple{ConcretePJRTArray{Float64, 3, 1, Reactant.Sharding.ShardInfo{Reactant.Sharding.NoSharding, Nothing}}}; sync::Bool, kwargs::@Kwargs{})
    @ Reactant.Compiler ~/Reactant.jl/src/Compiler.jl:2029
  [9] compile
    @ ~/Reactant.jl/src/Compiler.jl:2028 [inlined]
 [10] copy(bc::Base.Broadcast.Broadcasted{Base.Broadcast.ArrayStyle{ConcretePJRTArray}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}, Base.OneTo{Int64}}, typeof(identity), Tuple{ConcretePJRTArray{Float64, 3, 1, Reactant.Sharding.ShardInfo{…}}}})
    @ Reactant ~/Reactant.jl/src/ConcreteRArray.jl:391
 [11] copyto!(dest::ConcretePJRTArray{Float64, 3, 1, Reactant.Sharding.ShardInfo{…}}, src::Base.Broadcast.Broadcasted{Base.Broadcast.ArrayStyle{…}, Tuple{…}, typeof(identity), Tuple{…}})
    @ Reactant ~/Reactant.jl/src/ConcreteRArray.jl:450
 [12] materialize!
    @ ./broadcast.jl:883 [inlined]
 [13] materialize!
    @ ./broadcast.jl:880 [inlined]
 [14] set_to_field!(u::Field{…}, v::Field{…})
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:142
 [15] set!(u::Field{…}, v::Field{…})
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:37
 [16] set_to_function!(u::Field{Center, Center, Nothing, Nothing, OrthogonalSphericalShellGrid{…}, Tuple{…}, OffsetArrays.OffsetArray{…}, Float64, FieldBoundaryConditions{…}, Nothing, Oceananigans.Fields.FieldBoundaryBuffers{…}}, f::Function)
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:95
 [17] set!(u::Field{Center, Center, Nothing, Nothing, OrthogonalSphericalShellGrid{…}, Tuple{…}, OffsetArrays.OffsetArray{…}, Float64, FieldBoundaryConditions{…}, Nothing, Oceananigans.Fields.FieldBoundaryBuffers{…}}, f::Function)
    @ Oceananigans.Fields ~/.julia/packages/Oceananigans/OfIsz/src/Fields/set!.jl:35
 [18] materialize_immersed_boundary(grid::OrthogonalSphericalShellGrid{…}, ib::GridFittedBottom{…})
    @ OceananigansReactantExt.Grids ~/.julia/packages/Oceananigans/OfIsz/ext/OceananigansReactantExt/Grids/serial_grids.jl:15
 [19] ImmersedBoundaryGrid(grid::OrthogonalSphericalShellGrid{…}, ib::GridFittedBottom{…}; active_cells_map::Bool, active_z_columns::Bool)
    @ Oceananigans.ImmersedBoundaries ~/.julia/packages/Oceananigans/OfIsz/src/ImmersedBoundaries/immersed_boundary_grid.jl:47
 [20] macro expansion
    @ ~/.julia/juliaup/julia-1.11.4+0.x64.linux.gnu/share/julia/stdlib/v1.11/Profile/src/Profile.jl:59 [inlined]
 [21] gaussian_islands_tripolar_grid(arch::ReactantState, resolution::Int64, Nz::Int64)
    @ GordonBell25 ~/GB-25/src/data_free_ocean_climate_model.jl:22
 [22] data_free_ocean_climate_model_init(arch::ReactantState; resolution::Int64, Nz::Int64)
    @ GordonBell25 ~/GB-25/src/data_free_ocean_climate_model.jl:110
 [23] data_free_ocean_climate_model_init(arch::ReactantState)
    @ GordonBell25 ~/GB-25/src/data_free_ocean_climate_model.jl:102
 [24] top-level scope
    @ ~/GB-25/ocean-climate-simulation/ocean_climate_simulation_compile.jl:11
 [25] include(fname::String)
    @ Main ./sysimg.jl:38
 [26] top-level scope
    @ REPL[1]:1
Some type information was truncated. Use `show(err)` to see complete types.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions