update tdlambda

tdlambda.jl

 
+cd(@__DIR__)
 @everywhere begin
-    num_episodes        = 40
+    num_episodes        = 150
     α                   = 1 # Initial learning rate
     const ϵ             = 0.5 # Initial chance of choosing random action
     const decay_rate    = 0.99 # decay rate for learning rate and ϵ
@@ -13,11 +14,11 @@ end
 functions = [TDλlearning, TDOλlearning]
 
 for fun in functions
-    mc_runs = 3
-    λvec = linspace(0,1,10)
+    mc_runs = 1
+    λvec = linspace(0,1,5)
     λvecMC = repmat(λvec',mc_runs)[:]
     n = length(λvec)
-    res = pmap(λvecMC) do λ
+    @time res = pmap(λvecMC) do λ
         fun(num_episodes,α,λ)
     end
     rewards = getindex.(res,1)
@@ -28,11 +29,10 @@ for fun in functions
         mean(data,1)[:], std(data,1)[:]
     end
 
-    average_reward,ae = [mean(r.values[end-20:end]) for r in rewards] |> average
-    average_eval,aee = [mean(r.values) for r in rewards] |> average
-    max_eval,me = [maximum(r.values) for r in evals] |> average
-
-    scatter(λvec.+0.1rand(n,3)-0.05,[average_reward average_eval max_eval], xlabel="λ", lab=["Average reward" "Average eval" "Max eval"], title=string(fun),
-    yerror=[ae aee me])
+    # @show average_reward,ae = [mean(r.values[end-3:end]) for r in rewards] |> average
+    # average_eval,aee = [mean(r.values) for r in evals] |> average
+    # max_eval,me = [maximum(r.values) for r in evals] |> average
+    #
+    # scatter(λvec.+0.1rand(n,3)-0.05,[average_reward average_eval max_eval], xlabel="λ", lab=["Average reward" "Average eval" "Max eval"], title=string(fun),    yerror=[ae aee me])
 
 end

tdlambda_setup.jl

@@ -4,7 +4,7 @@ const env = GymEnv("CartPole-v0")
 typealias SARS Tuple{Vector{Float64},Int,Float64,Vector{Float64}}
 typealias V64 Vector{Float64}
 
-const ϕ = MultiUniformRBFE([linspace(-0.3,0.3) linspace(-2,2) linspace(-0.2,0.2) linspace(-3.2,3.2) linspace(0,1)], [4,4,4,4,2])
+const ϕ = MultiUniformRBFE([linspace(-0.3,0.3) linspace(-2,2) linspace(-0.2,0.2) linspace(-3.2,3.2) linspace(0,1)], [5,5,5,5,2])
 const P = length(ϕ(zeros(5)))
 
 type Qfun
@@ -44,6 +44,7 @@ const ϵpolicy       = ϵGreedyPolicy(ϵ, decay_rate)
 const policy        = GreedyPolicy()
 
 function TDλlearning(num_episodes,α,λ)
+    Q.θ .*= 0
     reward_history = ValueHistories.History(Float64)
     eval_history   = ValueHistories.History(Float64)
     for i = 1:num_episodes
@@ -52,7 +53,8 @@ function TDλlearning(num_episodes,α,λ)
         # α *= decay_rate # Decay the learning rate
         decay!(ϵpolicy) # Decay greedyness
         t = 0
-        for (t,(s::V64,a::Int,r::Float64,s1::V64)) in enumerate(take(ep,max_timesteps))
+        for (s::V64,a::Int,r::Float64,s1::V64) in take(ep,max_timesteps)
+            t += 1
             δ::Float64 = r + γ*max_a(Q, s1) - Q(s,a)
             E          = γ*λ*E + ϕ([s;a])::V64
             Q.θ       += α*δ*E
@@ -64,6 +66,7 @@ function TDλlearning(num_episodes,α,λ)
 end
 
 function TDOλlearning(num_episodes,α,λ)
+    Q.θ .*= 0
     reward_history = ValueHistories.History(Float64)
     eval_history   = ValueHistories.History(Float64)
     for i = 1:num_episodes
@@ -72,7 +75,8 @@ function TDOλlearning(num_episodes,α,λ)
         # α *= decay_rate # Decay the learning rate
         decay!(ϵpolicy) # Decay greedyness
         t, qs = 0, 0.
-        for (t,(s::V64,a::Int,r::Float64,s1::V64)) in enumerate(take(ep,max_timesteps))
+        for (s::V64,a::Int,r::Float64,s1::V64) in take(ep,max_timesteps)
+            t += 1
             if t == 1; qs = Q(s,a)::Float64; end
             qs1        = max_a(Q, s1)::Float64
             ϕs         = ϕ([s;a])::V64
@@ -97,39 +101,42 @@ function evaluate(i,eval_history)
     end
 end
 
-#
-# function TDAλlearning(num_episodes,α,λ)
-#     reward_history = ValueHistories.History(Float64)
-#     eval_history   = ValueHistories.History(Float64)
-#     sars           = Vector{SARS}(max_timesteps)
-#     for i = 1:num_episodes
-#         E  = zeros(P)   # Eligibility trace
-#         ep = Episode(env, ϵpolicy)
-#         α *= decay_rate # Decay the learning rate
-#         decay!(ϵpolicy) # Decay greedyness
-#         t = 0
-#         for (t,sarst::SARS) in enumerate(take(ep,max_timesteps))
-#             sars[t] = sarst
-#         end
-#         rewards = [sars[3] for sars in sars]
-#         Return = discounted_return(rewards,γ)
-#         for (t,(s,a,r,s1)::SARS) in enumerate(sars)
-#             δ::Float64 = r + γ*max_a(Q, s1) - Q(s,a)
-#             E          = γ*λ*E + ϕ([s;a])::Vector{Float64}
-#             Q.θ       += α*δ*E
-#         end
-#         push!(reward_history, i, ep.total_reward)
-#         if i % 10 == 0 # Evaluate without noise
-#             ep = Episode(env, policy)
-#             for sars in take(ep,max_timesteps)
-#             end
-#             println("Episode: $i, reward: $(ep.total_reward)")
-#             push!(eval_history, i, ep.total_reward)
-#         end
-#     end
-#     reward_history,eval_history
-# end
-#
+
+function TDAλlearning(num_episodes,α,λ)
+    reward_history = ValueHistories.History(Float64)
+    eval_history   = ValueHistories.History(Float64)
+    sars           = Vector{SARS}(max_timesteps)
+    for i = 1:num_episodes
+        E  = zeros(P)   # Eligibility trace
+        ep = Episode(env, ϵpolicy)
+        α *= decay_rate # Decay the learning rate
+        decay!(ϵpolicy) # Decay greedyness
+        t = 0
+        for sarst in take(ep,max_timesteps)
+            t += 1
+            sars[t] = sarst
+        end
+        rewards = [sars[3] for sars in sars]
+        Return = discounted_return(rewards,γ)
+        t = 0
+        for (s,a,r,s1) in sars
+            t += 1
+            δ::Float64 = r + γ*max_a(Q, s1) - Q(s,a)
+            E          = γ*λ*E + ϕ([s;a])::Vector{Float64}
+            Q.θ       += α*δ*E
+        end
+        push!(reward_history, i, ep.total_reward)
+        if i % 10 == 0 # Evaluate without noise
+            ep = Episode(env, policy)
+            for sars in take(ep,max_timesteps)
+            end
+            println("Episode: $i, reward: $(ep.total_reward)")
+            push!(eval_history, i, ep.total_reward)
+        end
+    end
+    reward_history,eval_history
+end
+
 #
 # function discounted_return(r,γ)
 #     l        = length(r)