``````

In :

import numpy as np
import pprint
import sys
if "../" not in sys.path:
sys.path.append("../")
from lib.envs.gridworld import GridworldEnv

``````
``````

In :

pp = pprint.PrettyPrinter(indent=2)
env = GridworldEnv()

``````
``````

In :

def policy_eval(policy, env, discount_factor=1.0, theta=0.00001):
"""
Evaluate a policy given an environment and a full description of the environment's dynamics.

Args:
policy: [S, A] shaped matrix representing the policy.
env: OpenAI env. env.P represents the transition probabilities of the environment.
env.P[s][a] is a (prob, next_state, reward, done) tuple.
theta: We stop evaluation once our value function change is less than theta for all states.
discount_factor: lambda discount factor.

Returns:
Vector of length env.nS representing the value function.
"""
V = np.zeros(env.nS)
while True:
delta = 0
# For each state, perform a "full backup"
for s in range(env.nS):
v = 0
# Look at the possible next actions
for a, action_prob in enumerate(policy[s]):
# For each action, look at the possible next states...
for  prob, next_state, reward, done in env.P[s][a]:
# Calculate the expected value
v += action_prob * prob * (reward + discount_factor * V[next_state])
# How much our value function changed (across any states)
delta = max(delta, np.abs(v - V[s]))
V[s] = v
# Stop evaluating once our value function change is below a threshold
if delta < theta:
break
return np.array(V)

``````
``````

In :

random_policy = np.ones([env.nS, env.nA]) / env.nA
v = policy_eval(random_policy, env)

``````
``````

In :

print("Value Function:")
print(v)
print("")

print("Reshaped Grid Value Function:")
print(v.reshape(env.shape))
print("")

``````
``````

Value Function:
[  0.         -13.99993529 -19.99990698 -21.99989761 -13.99993529
-17.9999206  -19.99991379 -19.99991477 -19.99990698 -19.99991379
-17.99992725 -13.99994569 -21.99989761 -19.99991477 -13.99994569   0.        ]

Reshaped Grid Value Function:
[[  0.         -13.99993529 -19.99990698 -21.99989761]
[-13.99993529 -17.9999206  -19.99991379 -19.99991477]
[-19.99990698 -19.99991379 -17.99992725 -13.99994569]
[-21.99989761 -19.99991477 -13.99994569   0.        ]]

``````
``````

In :

# Test: Make sure the evaluated policy is what we expected
expected_v = np.array([0, -14, -20, -22, -14, -18, -20, -20, -20, -20, -18, -14, -22, -20, -14, 0])
np.testing.assert_array_almost_equal(v, expected_v, decimal=2)

``````
``````

In [ ]:

``````