Q Learning

In [1]:

    

%matplotlib inline
from matplotlib import pyplot as plt
import gym
import numpy as np
from gym.envs.registration import register
from gym import wrappers
import shutil


    

In [2]:

    

# register(
#     id='FrozenLakeNotSlippery-v0',
#     entry_point='gym.envs.toy_text:FrozenLakeEnv',
#     kwargs={'map_name' : '4x4', 'is_slippery': False},
#     max_episode_steps=100,
#     reward_threshold=0.78, # optimum = .8196
# )

# env = gym.make('FrozenLakeNotSlippery-v0')


    

    




[2017-09-29 16:20:28,486] Making new env: FrozenLakeNotSlippery-v0

In [8]:

    

env = gym.make('FrozenLake-v0')
shutil.rmtree('/tmp/FrozenLake_01') 
env = wrappers.Monitor(env, '/tmp/FrozenLake_01')


    

    




[2017-09-29 15:15:26,473] Making new env: FrozenLake-v0
[2017-09-29 15:15:26,637] Creating monitor directory /tmp/FrozenLake_01

In [7]:

    

Q= np.zeros((env.observation_space.n, env.action_space.n)) #16x4
gamma=0.95
alpha=0.1
num_episodes = 1000
total_reward=0.0
for i in range(num_episodes):
    s = env.reset();
    done=False
    while done==False:
        if np.random.rand() < 0.2:
            a = np.random.randint(env.action_space.n)
        else:
            a = np.argmax(Q[s,:] )
        s_next,reward,done,info = env.step(a)
        total_reward+=reward
        if done == True:
            break
        #Q[s,a] = Q[s,a] +  lr*(r + y*np.max(Q[s1,:]) - Q[s,a])
        
        Q[s,a]= (1-alpha)*Q[s,a] + alpha*( reward + gamma*np.max(Q[s_next,:]) )
        s=s_next
print(total_reward)


    

    




0.0

In [6]:

    

print "Final Q-Table Values"
print Q


    

    




Final Q-Table Values
[[ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]
 [ 0.  0.  0.  0.]]

In [11]:

    

s = env.reset()
d=False
n=0
while d==False:
    n+=1
    a = np.argmax(Q[s,:])
    s,r,d,x = env.step(a)
    #print("%s %s %s %s"%(s,r,d,x))
env.render()
print n


    

    




[2017-09-29 15:15:59,519] Starting new video recorder writing to C:\tmp\FrozenLake_01\openaigym.video.0.12516.video100000.json






    




  (Left)
SFFF
FHFH
FFFH
HFFG
15

In [12]:

    

env.close()
gym.upload('/tmp/FrozenLake_01', api_key='sk_o9OoYpSkKamkW8MrKuHw')


    

    




[2017-09-29 15:16:00,266] Finished writing results. You can upload them to the scoreboard via gym.upload('C:\\tmp\\FrozenLake_01')
[2017-09-29 15:16:00,384] [FrozenLake-v0] Uploading 100001 episodes of training data






    




---------------------------------------------------------------------------
APIConnectionError                        Traceback (most recent call last)
<ipython-input-12-47b264500c83> in <module>()
      1 env.close()
----> 2 gym.upload('/tmp/FrozenLake_01', api_key='sk_o9OoYpSkKamkW8MrKuHw')

C:\Anaconda2\lib\site-packages\gym\scoreboard\api.pyc in upload(training_dir, algorithm_id, writeup, tags, benchmark_id, api_key, ignore_open_monitors, skip_videos)
     52             api_key=api_key,
     53             ignore_open_monitors=ignore_open_monitors,
---> 54             skip_videos=skip_videos,
     55         )
     56 

C:\Anaconda2\lib\site-packages\gym\scoreboard\api.pyc in _upload(training_dir, algorithm_id, writeup, benchmark_run_id, api_key, ignore_open_monitors, skip_videos)
    135             raise error.Error("Still have an open monitor on {}. You must run 'env.close()' before uploading.".format(', '.join(envs)))
    136 
--> 137     env_info, training_episode_batch, training_video = upload_training_data(training_dir, api_key=api_key, skip_videos=skip_videos)
    138     env_id = env_info['env_id']
    139     training_episode_batch_id = training_video_id = None

C:\Anaconda2\lib\site-packages\gym\scoreboard\api.pyc in upload_training_data(training_dir, api_key, skip_videos)
    191     # Do the relevant uploads
    192     if len(episode_lengths) > 0:
--> 193         training_episode_batch = upload_training_episode_batch(data_sources, episode_lengths, episode_rewards, episode_types, initial_reset_timestamps, timestamps, api_key, env_id=env_id)
    194     else:
    195         training_episode_batch = None

C:\Anaconda2\lib\site-packages\gym\scoreboard\api.pyc in upload_training_episode_batch(data_sources, episode_lengths, episode_rewards, episode_types, initial_reset_timestamps, timestamps, api_key, env_id)
    209 def upload_training_episode_batch(data_sources, episode_lengths, episode_rewards, episode_types, initial_reset_timestamps, timestamps, api_key=None, env_id=None):
    210     logger.info('[%s] Uploading %d episodes of training data', env_id, len(episode_lengths))
--> 211     file_upload = resource.FileUpload.create(purpose='episode_batch', api_key=api_key)
    212     file_upload.put({
    213         'data_sources': data_sources,

C:\Anaconda2\lib\site-packages\gym\scoreboard\client\resource.pyc in create(cls, api_key, **params)
    356         url = cls.class_path()
    357         response, api_key = requestor.request(
--> 358             'post', url, params=params)
    359         return convert_to_gym_object(response, api_key)
    360 

C:\Anaconda2\lib\site-packages\gym\scoreboard\client\api_requestor.pyc in request(self, method, url, params, headers)
     38     def request(self, method, url, params=None, headers=None):
     39         rbody, rcode, rheaders, my_api_key = self.request_raw(
---> 40             method.lower(), url, params, headers)
     41         resp = self.interpret_response(rbody, rcode, rheaders)
     42         return resp, my_api_key

C:\Anaconda2\lib\site-packages\gym\scoreboard\client\api_requestor.pyc in request_raw(self, method, url, params, supplied_headers)
    130 
    131         rbody, rcode, rheaders = self._client.request(
--> 132             method, abs_url, headers, post_data)
    133 
    134         return rbody, rcode, rheaders, my_api_key

C:\Anaconda2\lib\site-packages\gym\scoreboard\client\http_client.pyc in request(self, method, url, headers, post_data, files)
     62             # Would catch just requests.exceptions.RequestException, but can
     63             # also raise ValueError, RuntimeError, etc.
---> 64             self._handle_request_error(e, method, url)
     65 
     66         if logger.level <= logging.DEBUG:

C:\Anaconda2\lib\site-packages\gym\scoreboard\client\http_client.pyc in _handle_request_error(self, e, method, url)
     92                 err += " with no error message"
     93         msg = textwrap.fill(msg, width=140) + "\n\n(Network error: %s)" % (err,)
---> 94         raise error.APIConnectionError(msg)

APIConnectionError: Unexpected error communicating with OpenAI Gym (while calling post https://gym-api.openai.com/v1/files). If this problem persists, let us
know at gym@openai.com.

(Network error: ConnectionError: HTTPSConnectionPool(host='gym-api.openai.com', port=443): Max retries exceeded with url: /v1/files (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x000000000B9E6E80>: Failed to establish a new connection: [Errno 11001] getaddrinfo failed',)))

In [ ]: