``````

In :

import numpy as np
import pprint
import sys
if "../" not in sys.path:
sys.path.append("../")
from lib.envs.gridworld import GridworldEnv

``````
``````

In :

pp = pprint.PrettyPrinter(indent=2)
env = GridworldEnv()

``````
``````

In :

def value_iteration(env, theta=0.0001, discount_factor=1.0):
"""
Value Iteration Algorithm.

Args:
env: OpenAI environment. env.P represents the transition probabilities of the environment.
theta: Stopping threshold. If the value of all states changes less than theta
in one iteration we are done.
discount_factor: lambda time discount factor.

Returns:
A tuple (policy, V) of the optimal policy and the optimal value function.
"""

"""
Helper function to calculate the value for all action in a given state.

Args:
state: The state to consider (int)
V: The value to use as an estimator, Vector of length env.nS

Returns:
A vector of length env.nA containing the expected value of each action.
"""
A = np.zeros(env.nA)
for a in range(env.nA):
for prob, next_state, reward, done in env.P[state][a]:
A[a] += prob * (reward + discount_factor * V[next_state])
return A

V = np.zeros(env.nS)
while True:
# Stopping condition
delta = 0
# Update each state...
for s in range(env.nS):
# Do a one-step lookahead to find the best action
best_action_value = np.max(A)
# Calculate delta across all states seen so far
delta = max(delta, np.abs(best_action_value - V[s]))
# Update the value function
V[s] = best_action_value
# Check if we can stop
if delta < theta:
break

# Create a deterministic policy using the optimal value function
policy = np.zeros([env.nS, env.nA])
for s in range(env.nS):
# One step lookahead to find the best action for this state
best_action = np.argmax(A)
# Always take the best action
policy[s, best_action] = 1.0

return policy, V

``````
``````

In :

policy, v = value_iteration(env)

print("Policy Probability Distribution:")
print(policy)
print("")

print("Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):")
print(np.reshape(np.argmax(policy, axis=1), env.shape))
print("")

print("Value Function:")
print(v)
print("")

print("Reshaped Grid Value Function:")
print(v.reshape(env.shape))
print("")

``````
``````

Policy Probability Distribution:
[[ 1.  0.  0.  0.]
[ 0.  0.  0.  1.]
[ 0.  0.  0.  1.]
[ 0.  0.  1.  0.]
[ 1.  0.  0.  0.]
[ 1.  0.  0.  0.]
[ 1.  0.  0.  0.]
[ 0.  0.  1.  0.]
[ 1.  0.  0.  0.]
[ 1.  0.  0.  0.]
[ 0.  1.  0.  0.]
[ 0.  0.  1.  0.]
[ 1.  0.  0.  0.]
[ 0.  1.  0.  0.]
[ 0.  1.  0.  0.]
[ 1.  0.  0.  0.]]

Reshaped Grid Policy (0=up, 1=right, 2=down, 3=left):
[[0 3 3 2]
[0 0 0 2]
[0 0 1 2]
[0 1 1 0]]

Value Function:
[ 0. -1. -2. -3. -1. -2. -3. -2. -2. -3. -2. -1. -3. -2. -1.  0.]

Reshaped Grid Value Function:
[[ 0. -1. -2. -3.]
[-1. -2. -3. -2.]
[-2. -3. -2. -1.]
[-3. -2. -1.  0.]]

``````
``````

In [ ]:

``````