In [1]:

    

%matplotlib inline

import gym
import matplotlib
import numpy as np
import sys

from collections import defaultdict
if "../" not in sys.path:
  sys.path.append("../") 
from lib.envs.blackjack import BlackjackEnv
from lib import plotting

matplotlib.style.use('ggplot')


    

In [2]:

    

env = BlackjackEnv()


    

In [3]:

    

def make_epsilon_greedy_policy(Q, epsilon, nA):
    """
    Creates an epsilon-greedy policy based on a given Q-function and epsilon.
    
    Args:
        Q: A dictionary that maps from state -> action-values.
            Each value is a numpy array of length nA (see below)
        epsilon: The probability to select a random action . float between 0 and 1.
        nA: Number of actions in the environment.
    
    Returns:
        A function that takes the observation as an argument and returns
        the probabilities for each action in the form of a numpy array of length nA.
    
    """
    def policy_fn(observation):
        probs = np.ones(nA, dtype=float) * epsilon / nA
        best_a = np.argmax(Q[observation])
        probs[best_a] = 1.0 - epsilon + epsilon / nA
        return probs
    return policy_fn


    

In [4]:

    

def mc_control_epsilon_greedy(env, num_episodes, discount_factor=1.0, epsilon=0.1):
    """
    Monte Carlo Control using Epsilon-Greedy policies.
    Finds an optimal epsilon-greedy policy.
    
    Args:
        env: OpenAI gym environment.
        num_episodes: Nubmer of episodes to sample.
        discount_factor: Lambda discount factor.
        epsilon: Chance the sample a random action. Float betwen 0 and 1.
    
    Returns:
        A tuple (Q, policy).
        Q is a dictionary mapping state -> action values.
        policy is a function taht takes an observation as an argument and returns
        action probabilities
    """
    
    # Keeps track of sum and count of returns for each state
    # to calculate an average. We could use an array to save all
    # returns (like in the book) but that's memory inefficient.
    returns_sum = defaultdict(float)
    returns_count = defaultdict(float)
    
    # The final action-value function.
    # A nested dictionary that maps state -> (action -> action-value).
    Q = defaultdict(lambda: np.zeros(env.action_space.n))
    
    # The policy we're following
    policy = make_epsilon_greedy_policy(Q, epsilon, env.action_space.n)
    
    for _ in range(num_episodes):
        done = False
        episode = []
        s = env.reset()
        while not done:
            probs = policy(s)
            action = np.random.choice(np.arange(len(probs)), p=probs)
            ns, reward, done, info = env.step(action)
            episode.append((s, action, reward))
            s = ns
            
        sa_history = list(map(lambda x: (x[0], x[1]), episode))
        for s, a in set(sa_history):
            first_pos = next(i for i,(xs,xa) in enumerate(sa_history) 
                             if xs == s and xa == a)
            G = sum([x[2] for x in episode[first_pos:]])
            returns_sum[(s,a)] += G
            returns_count[(s,a)] += 1.0
            Q[s][a] = returns_sum[(s,a)] / returns_count[(s,a)]
    
    return Q, policy


    

In [5]:

    

Q, policy = mc_control_epsilon_greedy(env, num_episodes=500000, epsilon=0.1)


    

In [6]:

    

# For plotting: Create value function from action-value function
# by picking the best action at each state
V = defaultdict(float)
for state, actions in Q.items():
    action_value = np.max(actions)
    V[state] = action_value
plotting.plot_value_function(V, title="Optimal Value Function")


    

In [ ]: