Using POMDPs.jl and running the lightdark.ipynb

[BPAServices]

Folks,
trying to run this https://nbviewer.jupyter.org/url/zachary.sunberg.net/assets/notebooks/pomdps_lightdark_demo.ipynb#
Changed the importall to the following:
using Pkg; Pkg.add("POMDPs")
using Pkg; Pkg.add("POMDPToolbox")
using Pkg; Pkg.add("Distributions")
using Pkg; Pkg.add("Parameters")
using Pkg; Pkg.add("Plots")
using Pkg; Pkg.add("StaticArrays")

Assuming all the packages loaded properly:
Resolving package versions...
No Changes to C:\Users\cveda\.julia\environments\v1.6\Project.toml
No Changes to C:\Users\cveda\.julia\environments\v1.6\Manifest.toml
Resolving package versions...
No Changes to C:\Users\cveda\.julia\environments\v1.6\Project.toml
No Changes to C:\Users\cveda\.julia\environments\v1.6\Manifest.toml
Resolving package versions...
No Changes to C:\Users\cveda\.julia\environments\v1.6\Project.toml
No Changes to C:\Users\cveda\.julia\environments\v1.6\Manifest.toml
Resolving package versions...
No Changes to C:\Users\cveda\.julia\environments\v1.6\Project.toml
No Changes to C:\Users\cveda\.julia\environments\v1.6\Manifest.toml
Resolving package versions...
No Changes to C:\Users\cveda\.julia\environments\v1.6\Project.toml
No Changes to C:\Users\cveda\.julia\environments\v1.6\Manifest.toml
Resolving package versions...
No Changes to C:\Users\cveda\.julia\environments\v1.6\Project.toml
No Changes to C:\Users\cveda\.julia\environments\v1.6\Manifest.toml

Now I'm running the next step which is

@with_kw struct SimpleLightDark <: POMDPs.POMDP{Int,Int,Float64}
    discount::Float64       = 0.95
    correct_r::Float64      = 100.0
    incorrect_r::Float64    = -100.0
    light_loc::Int          = 10
    radius::Int             = 60
end
discount(p::SimpleLightDark) = p.discount
isterminal(p::SimpleLightDark, s::Number) = !(s in -p.radius:p.radius)

const ACTIONS = [-10, -1, 0, 1, 10]
actions(p::SimpleLightDark) = ACTIONS
n_actions(p::SimpleLightDark) = length(actions(p))
const ACTION_INDS = Dict(a=>i for (i,a) in enumerate(actions(SimpleLightDark())))
action_index(p::SimpleLightDark, a::Int) = ACTION_INDS[a]

states(p::SimpleLightDark) = -p.radius:p.radius + 1
n_states(p::SimpleLightDark) = length(states(p))
state_index(p::SimpleLightDark, s::Int) = s+p.radius+1

function transition(p::SimpleLightDark, s::Int, a::Int) 
    if a == 0
        return SparseCat(SVector(p.radius+1), SVector(1.0))
    else
        return SparseCat(SVector(clamp(s+a, -p.radius, p.radius)), SVector(1.0))
    end
end

observation(p::SimpleLightDark, sp) = Normal(sp, abs(sp - p.light_loc) + 0.0001)

function reward(p::SimpleLightDark, s, a)
    if a == 0
        return s == 0 ? p.correct_r : p.incorrect_r
    else
        return -1.0
    end
end

function initial_state_distribution(p::SimpleLightDark)
    ps = ones(2*div(p.radius,2)+1)
    ps /= length(ps)
    return SparseCat(div(-p.radius,2):div(p.radius,2), ps)
end;

and it gives me the following error:
LoadError: UndefVarError: @with_kw not defined
in expression starting at In[5]:1

Stacktrace:
[1] top-level scope
@ :0
[2] eval
@ .\boot.jl:360 [inlined]
[3] include_string(mapexpr::typeof(REPL.softscope), mod::Module, code::String, filename::String)
@ Base .\loading.jl:1116

[rejuvyesh]

You likely need to do using Parameters.

[cvedara]

using POMDPs, QuickPOMDPs, POMDPModelTools, POMDPSimulators, QMDP
#using QuickPOMDPs
#import Pkg;Pkg.add("POMDPModelTools")
using Distributions
#=
The simple 1-D light dark problem from https://arxiv.org/pdf/1709.06196v6.pdf, section 5.2, or https://slides.com/zacharysunberg/defense-4#/39
=#
r = 60
light_loc = 10
simple_lightdark = QuickPOMDP(
states = -r:r+1, # r+1 is a terminal state
actions = [-10, -1, 0, 1, 10],
discount = 0.95,
initialstate = POMDPModelTools.Uniform(div(-r,2):div(r,2)),
isterminal = s -> !(s in -r:r),
obstype = Float64,
transition = function (s, a)
if a == 0
return Deterministic(r+1)
else
return Deterministic(clamp(s+a, -r, r))
end
end,

observation = (s, a, sp) -> Normal(sp, abs(sp - light_loc) + 0.0001),

reward = function (s, a)
    if a == 0
        return s == 0 ? 100 : -100
    else
        return -1.0
    end
end

)
solver = QMDPSolver()
policy = solve(solver, simple_lightdark)
rsum = 0.0
for (s,b,a,o,r) in stepthrough(simple_lightdark, policy, "s,b,a,o,r", max_steps=12)
println("s: $s, b: $([pdf(b,s) for s in states(simple_lightdark)]), a: $a, o: $o")
println("rsum: $rsum, r: $r")
global rsum += r
end
println("Undiscounted reward was $rsum.")

Added a solver and a policy and went through the loop to get an rsum

[cvedara]

Got an error trying to use the plotting code Zachary suggested and the error is following:
HistoryRecorder(; rng, eps, max_steps, capture_exception, show_progress) at C:\Users\cveda.julia\packages\POMDPSimulators\KbSch\src\history_recorder.jl:37 got unsupported keyword arguments "initial_state", "initial_obs", "updater"
HistoryRecorder(::AbstractRNG, ::Bool, ::Bool, ::Any, ::Any) at C:\Users\cveda.julia\packages\POMDPSimulators\KbSch\src\history_recorder.jl:25 got unsupported keyword arguments "initial_state", "rng", "initial_obs", "max_steps", "updater"
HistoryRecorder(::Any, ::Any, ::Any, ::Any, ::Any) at C:\Users\cveda.julia\packages\POMDPSimulators\KbSch\src\history_recorder.jl:25 got unsupported keyword arguments "initial_state", "rng", "initial_obs", "max_steps", "updater"

here is the plotting code:
h = sim(p, updater=pf, initial_state=1, initial_obs=initial_state_distribution(p), max_steps=80, rng=rng) do b
return rand(rng, [-1,1])
end

[zsunberg]

Hi @cvedara , sorry for the delayed response - it seems that we were all busy.

I think the error is because the keyword arguments should not have a space in them https://juliapomdp.github.io/POMDPSimulators.jl/stable/sim/#sim-function

[cvedara]

Hi Zachary,
Thanks for the response. I'm trying to do a simple historyrecorder and simulate and plot and somehow it points back to my initial_state_distribution in the QuickPOMDP.
hr = HistoryRecorder(max_steps=100)
policy = solve(solver, p)
h = simulate(hr, p, policy)
plothist(p, h)

MethodError: no method matching initial_state_distribution(::Int64, ::Int64, ::Random._GLOBAL_RNG)
Closest candidates are:
initial_state_distribution(::Any) at In[72]:27

Stacktrace:
[1] gen(m::QuickPOMDP{UUID("3661c15d-0736-4e39-8a91-116edf1ebf4e"), Int64, Int64, Float64, NamedTuple{(:stateindex, :isterminal, :gen, :states, :actions, :obstype, :observation, :actionindex, :initialstate, :transition, :reward, :discount, :initialobs), Tuple{Dict{Int64, Int64}, var"#133#137", typeof(initial_state_distribution), UnitRange{Int64}, Vector{Int64}, DataType, var"#135#139", Dict{Int64, Int64}, POMDPModelTools.Uniform{Set{Int64}}, var"#134#138", var"#136#140", Float64, POMDPModelTools.Uniform{Set{Int64}}}}}, s::Int64, a::Int64, rng::Random._GLOBAL_RNG)

Here is my QuickPOMDP constructor:
simple_lightdark = QuickPOMDP(
states = -r:r+1, # r+1 is a terminal state
actions = [-10, -1, 0, 1, 10],
discount = 0.95,
initialstate = POMDPModelTools.Uniform(div(-r,2):div(r,2)),
initialobs=POMDPModelTools.Uniform(div(-r,2):div(r,2)),
isterminal = s -> !(s in -r:r),
obstype = Float64,
transition = function (s, a)
if a == 0
return Deterministic(r+1)
else
return Deterministic(clamp(s+a, -r, r))
end
end,

observation = (s, a, sp) -> Normal(sp, abs(sp - light_loc) + 0.0001),

reward = function (s, a, sp, o)

reward = function (s, a)
    if a == 0
        return s == 0 ? 100 : -100
    else
        return -1.0
    end
end,
function initial_state_distribution(p)
    ps = ones(2*div(radius,2)+1)
    ps /= length(ps)
    return SparseCat(div(-radius,2):div(radius,2), ps)
end

)

Can you see what is wrong here.

[zsunberg]

Yes, right now initial_state_distribution is treated as a positional argument instead of a keyword argument, so the QuickPOMDP constructor is interpreting it incorrectly. I would recommend using only keyword arguments in the QuickPOMDP constructor.

In any case, initial_state_distribution is no longer used. Instead initialstate should be used to specify the initial state distribution.