IntervalMDP.jl/src/value.jl at b1717001c9dcd7cf4e47cd308a2a97d5da2f6d74 · Zinoex/IntervalMDP.jl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
@enum IntervalMode Upper Lower
isupper(mode::IntervalMode) = mode == Upper
islower(mode::IntervalMode) = mode == Lower

abstract type ValueFunction end

struct StateValueFunction{R, A1 <: AbstractArray{R}, A2 <: AbstractArray{R}} <: ValueFunction
    previous::A1
    current::A1
    intermediate_state_action_value::A2
    interval::IntervalMode
end

function StateValueFunction(problem::AbstractIntervalMDPProblem)
    mp = system(problem)
    previous = arrayfactory(mp, valuetype(mp), state_values(mp))
    previous .= zero(valuetype(mp))
    current = copy(previous)

    dim = (action_values(mp)..., state_values(mp)...)
    # concat gives shape: (a1, a2) , (s1, s2) => (a1, a2, s1, s2)
    # (a, s) to access a more frequently due to column major
    # TODO: works for IMDP, need to check for fIMDP
    intermediate_state_action_value = arrayfactory(mp, valuetype(mp), dim)
    intermediate_state_action_value .= zero(valuetype(mp))

    return StateValueFunction(previous, current, intermediate_state_action_value, Lower)
end

function StateValueFunction(problem::AbstractIntervalMDPProblem, mode::IntervalMode)
    mp = system(problem)
    previous = arrayfactory(mp, valuetype(mp), state_values(mp))
    previous .= zero(valuetype(mp))
    current = copy(previous)

    dim = (action_values(mp)..., state_values(mp)...)
    # concat gives shape: (a1, a2) , (s1, s2) => (a1, a2, s1, s2)
    # (a, s) to access a more frequently due to column major
    # TODO: works for IMDP, need to check for fIMDP
    intermediate_state_action_value = arrayfactory(mp, valuetype(mp), dim)
    intermediate_state_action_value .= zero(valuetype(mp))

    return StateValueFunction(previous, current, intermediate_state_action_value, mode)
end

function lastdiff!(V::StateValueFunction{R}) where {R}
    # Reuse prev to store the latest difference
    V.previous .-= V.current
    rmul!(V.previous, -one(R))

    return V.previous
end

function nextiteration!(V::StateValueFunction)
    copy!(V.previous, V.current)

    return V
end

islower(V::StateValueFunction) = islower(V.interval)
isupper(V::StateValueFunction) = isupper(V.interval)


struct StateActionValueFunction{R, A1 <: AbstractArray{R}, A2 <: AbstractArray{R}} <: ValueFunction
    previous::A1
    current::A1
    intermediate_state_value::A2
    interval::IntervalMode
end

function StateActionValueFunction(problem::AbstractIntervalMDPProblem)
    mp = system(problem)
    dim = (action_values(mp)..., state_values(mp)...)
    # TODO: works for IMDP, need to check for fIMDP
    previous = arrayfactory(mp, valuetype(mp), dim)
    previous .= zero(valuetype(mp))
    current = copy(previous)

    intermediate_state_value = arrayfactory(mp, valuetype(mp), state_values(mp))
    intermediate_state_value .= zero(valuetype(mp))

    return StateActionValueFunction(previous, current, intermediate_state_value, Lower)
end

function StateActionValueFunction(problem::AbstractIntervalMDPProblem, mode::IntervalMode)
    mp = system(problem)
    dim = (action_values(mp)..., state_values(mp)...)
    # TODO: works for IMDP, need to check for fIMDP
    previous = arrayfactory(mp, valuetype(mp), dim)
    previous .= zero(valuetype(mp))
    current = copy(previous)

    intermediate_state_value = arrayfactory(mp, valuetype(mp), state_values(mp))
    intermediate_state_value .= zero(valuetype(mp))

    return StateActionValueFunction(previous, current, intermediate_state_value, mode)
end


function lastdiff!(V::StateActionValueFunction{R}) where {R}
    # Reuse prev to store the latest difference
    V.previous .-= V.current
    rmul!(V.previous, -one(R))

    return V.previous
end

function nextiteration!(V::StateActionValueFunction)
    copy!(V.previous, V.current)

    return V
end

islower(V::StateActionValueFunction) = islower(V.interval)
isupper(V::StateActionValueFunction) = isupper(V.interval)


struct IntervalValueFunction{V <: ValueFunction} <: ValueFunction
    lower::V
    upper::V
end

lower(V::IntervalValueFunction) = V.lower
upper(V::IntervalValueFunction) = V.upper

function lastdiff!(V::IntervalValueFunction)
    return (lastdiff!(V.lower), lastdiff!(V.upper))
end

function nextiteration!(V::IntervalValueFunction)
    nextiteration!(V.lower)
    nextiteration!(V.upper)

    return V
end

function gap(V::IntervalValueFunction)
    return abs.(V.lower.current .- V.upper.current)
end

function initialize!(value_function::IntervalValueFunction, prop::AbstractReachability)
    initialize!(value_function.lower, prop, Val(false))
    initialize!(value_function.upper, prop, Val(true))
end


#################
# Algorithms    #
#################
construct_value_function(::RobustValueIteration, problem) = StateValueFunction(problem)
construct_value_function(::IntervalValueIteration, problem) = IntervalValueFunction(lower=StateValueFunction(problem), upper=StateValueFunction(problem))
construct_value_function(::GeneralizedSamplingbasedRobustDynamicProgramming, problem) = IntervalValueFunction(lower=StateActionValueFunction(problem), upper=StateActionValueFunction(problem))