In [1]:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
In [2]:
from dqn_learning import *
# import cPickle
# %load_ext autoreload
# %autoreload 2
%pylab inline
# import numpy as np
[2016-08-22 12:12:04,706] Site environment registry incorrect: Scoreboard did not register all envs: set(['AcrobotContinuous-v0'])
Populating the interactive namespace from numpy and matplotlib
WARNING: pylab import has clobbered these variables: ['copy']
`%matplotlib` prevents importing * from pylab and numpy
In [3]:
car = q_learning(gamma=0.99,
# lambda_=0.9,
# init_epsilon = 0.0,
# update_epsilon = False,
init_epsilon = 1.0,
end_epsilon = 0.1,
update_epsilon = True,
exploration_decrease_length = 1e6,
# N_0= 50,
environment = 'CartPole-v0',
plot_resolution = 30,
nn_size_hidden = [400,300], # [100,100] --> 0.01, [100,100,100] --> similar [400,300]
nn_batch_size = 32,
nn_learning_rate = 2.5e-4,
replay_memory_size = 3e5, # larger than nn_batch_size*1e3 or doesn't train
descent_method = 'rmsprop',
dropout_keep_prob = 0.9,
ema_decay_rate = 0.999,
train_frequency = 4.0,
from_pixels = True,
reg_weight = 0.0,
do_pretrain = False,
pretrain_steps = 0
)
[2016-08-22 12:12:07,230] Making new env: CartPole-v0
conv 128 8 8 2
weights std = 0.125
[8, 8, 2, 32]
bias std = 0.0
conv 512 4 4 32
weights std = 0.0625
[4, 4, 32, 64]
bias std = 0.0
conv 576 3 3 64
weights std = 0.0589255650989
[3, 3, 64, 64]
bias std = 0.0
fc 7744
weights std = 0.0160706086633
bias std = 0.0
fc 512
weights std = 0.0441941738242
bias std = 0.0
Q-learning configured in nn graph
descending with rmsprop
using environment CartPole-v0
qnn target q-learning False False
In [4]:
car.deepq_learning(num_iter = 1000000, max_steps = 500000, max_learning_steps=1e6)#, reset_replay_memory=True)
Episode 1056 total samples 10011 train steps 2
Length 23
Episode 1297 total samples 12513 train steps 315
Length 9
Episode 1551 total samples 15016 train steps 627
Length 16
Episode 1819 total samples 17521 train steps 941
Length 31
Episode 2083 total samples 20023 train steps 1253
Length 36
Episode 2348 total samples 22531 train steps 1567
Length 17
exploration 0.977496400001
/usr/lib/pymodules/python2.7/matplotlib/collections.py:548: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
if self._edgecolors == 'face':
Episode 2614 total samples 25047 train steps 1881
Length 72
Episode 2882 total samples 27550 train steps 2194
Length 8
Episode 3150 total samples 30058 train steps 2508
Length 28
Episode 3419 total samples 32567 train steps 2821
Length 26
Episode 3656 total samples 35077 train steps 3135
Length 33
Episode 3905 total samples 37589 train steps 3449
Length 28
Episode 4161 total samples 40097 train steps 3763
Length 23
Episode 4435 total samples 42602 train steps 4076
Length 14
Episode 4704 total samples 45123 train steps 4391
Length 56
Episode 4960 total samples 47629 train steps 4704
Length 13
exploration 0.954995500001
Episode 5215 total samples 50132 train steps 5017
Length 8
Episode 5468 total samples 52633 train steps 5330
Length 10
Episode 5713 total samples 55134 train steps 5642
Length 17
Episode 5952 total samples 57638 train steps 5955
Length 18
Episode 6199 total samples 60140 train steps 6268
Length 23
Episode 6450 total samples 62648 train steps 6581
Length 16
Episode 6711 total samples 65162 train steps 6896
Length 29
Episode 6948 total samples 67664 train steps 7208
Length 15
Episode 7198 total samples 70175 train steps 7522
Length 24
Episode 7451 total samples 72679 train steps 7835
Length 12
exploration 0.932486500002
Episode 7703 total samples 75184 train steps 8148
Length 18
Episode 7947 total samples 77688 train steps 8461
Length 22
Episode 8194 total samples 80194 train steps 8775
Length 12
Episode 8445 total samples 82705 train steps 9089
Length 31
Episode 8693 total samples 85208 train steps 9401
Length 10
Episode 8932 total samples 87710 train steps 9714
Length 12
Episode 9185 total samples 90212 train steps 10027
Length 13
Episode 9426 total samples 92720 train steps 10340
Length 21
Episode 9664 total samples 95223 train steps 10653
Length 22
Episode 9900 total samples 97732 train steps 10967
Length 17
exploration 0.909981100003
Episode 10149 total samples 100236 train steps 11280
Length 26
Episode 10376 total samples 102741 train steps 11593
Length 24
Episode 10601 total samples 105242 train steps 11906
Length 21
Episode 10836 total samples 107750 train steps 12219
Length 31
Episode 11063 total samples 110262 train steps 12533
Length 41
Episode 11289 total samples 112764 train steps 12846
Length 14
Episode 11523 total samples 115289 train steps 13162
Length 88
Episode 11764 total samples 117797 train steps 13475
Length 22
Episode 12000 total samples 120305 train steps 13789
Length 25
Episode 12228 total samples 122812 train steps 14102
Length 26
exploration 0.887474800004
Episode 12433 total samples 125326 train steps 14416
Length 35
Episode 12672 total samples 127833 train steps 14730
Length 15
Episode 12900 total samples 130343 train steps 15043
Length 24
Episode 13129 total samples 132848 train steps 15356
Length 28
Episode 13358 total samples 135361 train steps 15671
Length 28
Episode 13583 total samples 137869 train steps 15984
Length 22
Episode 13803 total samples 140378 train steps 16298
Length 17
Episode 14027 total samples 142890 train steps 16612
Length 25
Episode 14230 total samples 145406 train steps 16926
Length 44
Episode 14451 total samples 147908 train steps 17239
Length 31
exploration 0.864955900004
Episode 14688 total samples 150413 train steps 17552
Length 49
Episode 14915 total samples 152926 train steps 17866
Length 65
Episode 15121 total samples 155436 train steps 18180
Length 33
Episode 15359 total samples 157941 train steps 18493
Length 23
Episode 15573 total samples 160446 train steps 18806
Length 25
Episode 15789 total samples 162952 train steps 19119
Length 14
Episode 16009 total samples 165459 train steps 19433
Length 17
Episode 16236 total samples 167961 train steps 19746
Length 47
Episode 16454 total samples 170470 train steps 20059
Length 35
Episode 16666 total samples 172973 train steps 20372
Length 21
exploration 0.842454100005
Episode 16896 total samples 175481 train steps 20686
Length 16
Episode 17107 total samples 177983 train steps 20998
Length 35
Episode 17321 total samples 180495 train steps 21312
Length 28
Episode 17529 total samples 183012 train steps 21627
Length 51
Episode 17751 total samples 185519 train steps 21940
Length 23
Episode 17964 total samples 188023 train steps 22253
Length 21
Episode 18167 total samples 190531 train steps 22567
Length 15
Episode 18387 total samples 193046 train steps 22881
Length 44
Episode 18592 total samples 195561 train steps 23196
Length 61
Episode 18794 total samples 198067 train steps 23509
Length 23
exploration 0.819952300006
Episode 19003 total samples 200577 train steps 23823
Length 22
Episode 19211 total samples 203094 train steps 24137
Length 41
Episode 19431 total samples 205627 train steps 24454
Length 107
Episode 19654 total samples 208133 train steps 24767
Length 22
Episode 19872 total samples 210638 train steps 25080
Length 19
Episode 20078 total samples 213145 train steps 25394
Length 54
Episode 20290 total samples 215661 train steps 25708
Length 40
Episode 20494 total samples 218163 train steps 26021
Length 39
Episode 20709 total samples 220669 train steps 26334
Length 58
Episode 20917 total samples 223174 train steps 26647
Length 18
exploration 0.797449600007
Episode 21121 total samples 225675 train steps 26960
Length 10
Episode 21318 total samples 228180 train steps 27273
Length 45
Episode 21509 total samples 230682 train steps 27586
Length 9
Episode 21720 total samples 233193 train steps 27900
Length 42
Episode 21916 total samples 235698 train steps 28213
Length 40
Episode 22118 total samples 238241 train steps 28531
Length 85
Episode 22324 total samples 240743 train steps 28843
Length 30
Episode 22535 total samples 243250 train steps 29157
Length 13
Episode 22735 total samples 245757 train steps 29470
Length 33
Episode 22957 total samples 248266 train steps 29784
Length 29
exploration 0.774945100007
Episode 23157 total samples 250771 train steps 30097
Length 26
Episode 23359 total samples 253283 train steps 30411
Length 23
Episode 23552 total samples 255791 train steps 30724
Length 21
Episode 23746 total samples 258305 train steps 31039
Length 49
Episode 23944 total samples 260813 train steps 31352
Length 60
Episode 24131 total samples 263321 train steps 31666
Length 17
Episode 24318 total samples 265827 train steps 31979
Length 32
Episode 24496 total samples 268332 train steps 32292
Length 14
Episode 24687 total samples 270834 train steps 32605
Length 15
Episode 24860 total samples 273346 train steps 32919
Length 89
exploration 0.752415400008
Episode 25058 total samples 275861 train steps 33233
Length 45
Episode 25252 total samples 278381 train steps 33548
Length 72
Episode 25445 total samples 280882 train steps 33861
Length 25
Episode 25636 total samples 283388 train steps 34174
Length 28
Episode 25807 total samples 285894 train steps 34487
Length 13
Episode 25989 total samples 288398 train steps 34800
Length 29
Episode 26182 total samples 290910 train steps 35114
Length 24
Episode 26368 total samples 293411 train steps 35427
Length 57
Episode 26555 total samples 295912 train steps 35739
Length 16
Episode 26746 total samples 298416 train steps 36052
Length 11
exploration 0.729910900009
Episode 26921 total samples 300938 train steps 36368
Length 55
Episode 27104 total samples 303444 train steps 36681
Length 16
Episode 27296 total samples 305949 train steps 36994
Length 16
Episode 27473 total samples 308450 train steps 37307
Length 72
Episode 27655 total samples 310955 train steps 37620
Length 24
Episode 27841 total samples 313457 train steps 37933
Length 26
Episode 28026 total samples 315978 train steps 38248
Length 70
Episode 28207 total samples 318488 train steps 38561
Length 33
Episode 28389 total samples 321003 train steps 38876
Length 50
Episode 28570 total samples 323543 train steps 39193
Length 99
exploration 0.70740910001
Episode 28749 total samples 326044 train steps 39506
Length 17
Episode 28925 total samples 328578 train steps 39823
Length 68
Episode 29109 total samples 331081 train steps 40136
Length 28
Episode 29290 total samples 333596 train steps 40450
Length 29
Episode 29470 total samples 336103 train steps 40763
Length 105
Episode 29641 total samples 338609 train steps 41077
Length 48
Episode 29801 total samples 341118 train steps 41390
Length 62
Episode 29983 total samples 343636 train steps 41705
Length 131
Episode 30153 total samples 346142 train steps 42018
Length 18
Episode 30331 total samples 348676 train steps 42335
Length 78
exploration 0.68490820001
Episode 30501 total samples 351186 train steps 42649
Length 29
Episode 30667 total samples 353690 train steps 42962
Length 74
Episode 30832 total samples 356205 train steps 43276
Length 39
Episode 30990 total samples 358714 train steps 43590
Length 30
Episode 31155 total samples 361223 train steps 43903
Length 47
Episode 31316 total samples 363741 train steps 44218
Length 46
Episode 31476 total samples 366251 train steps 44532
Length 22
Episode 31647 total samples 368766 train steps 44846
Length 31
Episode 31805 total samples 371271 train steps 45159
Length 13
Episode 31978 total samples 373778 train steps 45473
Length 55
exploration 0.662403700011
Episode 32135 total samples 376288 train steps 45786
Length 24
Episode 32284 total samples 378796 train steps 46100
Length 61
Episode 32453 total samples 381298 train steps 46413
Length 14
Episode 32611 total samples 383810 train steps 46727
Length 34
Episode 32769 total samples 386316 train steps 47040
Length 44
Episode 32935 total samples 388817 train steps 47353
Length 9
Episode 33094 total samples 391320 train steps 47665
Length 56
Episode 33265 total samples 393822 train steps 47978
Length 13
Episode 33427 total samples 396348 train steps 48294
Length 55
Episode 33592 total samples 398857 train steps 48608
Length 41
exploration 0.639899200012
Episode 33739 total samples 401370 train steps 48922
Length 54
Episode 33893 total samples 403883 train steps 49236
Length 52
Episode 34053 total samples 406407 train steps 49551
Length 74
Episode 34214 total samples 408912 train steps 49864
Length 11
Episode 34382 total samples 411423 train steps 50178
Length 22
Episode 34538 total samples 413925 train steps 50491
Length 46
Episode 34685 total samples 416428 train steps 50804
Length 20
Episode 34843 total samples 418931 train steps 51117
Length 15
Episode 35002 total samples 421432 train steps 51429
Length 14
Episode 35152 total samples 423936 train steps 51742
Length 14
exploration 0.617391100013
Episode 35298 total samples 426441 train steps 52056
Length 10
Episode 35437 total samples 428952 train steps 52369
Length 46
Episode 35588 total samples 431465 train steps 52684
Length 32
Episode 35729 total samples 433977 train steps 52998
Length 30
Episode 35878 total samples 436480 train steps 53310
Length 12
Episode 36034 total samples 438985 train steps 53624
Length 48
Episode 36179 total samples 441489 train steps 53937
Length 12
Episode 36333 total samples 444010 train steps 54252
Length 46
Episode 36484 total samples 446511 train steps 54564
Length 50
Episode 36629 total samples 449018 train steps 54878
Length 32
exploration 0.594887500013
Episode 36772 total samples 451526 train steps 55191
Length 102
Episode 36927 total samples 454035 train steps 55505
Length 36
Episode 37063 total samples 456559 train steps 55820
Length 67
Episode 37208 total samples 459067 train steps 56134
Length 16
Episode 37347 total samples 461569 train steps 56447
Length 18
Episode 37490 total samples 464092 train steps 56762
Length 46
Episode 37624 total samples 466603 train steps 57076
Length 91
Episode 37762 total samples 469121 train steps 57391
Length 51
Episode 37894 total samples 471646 train steps 57706
Length 53
[2016-08-22 13:37:35,072] Observation '[-2.40424456 -1.83309912 0.08360479 -0.05389048]' is not contained within observation space 'Box(4,)'.
Episode 38034 total samples 474180 train steps 58023
Length 126
exploration 0.572384800014
Episode 38168 total samples 476709 train steps 58339
Length 69
Episode 38310 total samples 479214 train steps 58652
Length 32
Episode 38455 total samples 481722 train steps 58966
Length 15
Episode 38598 total samples 484236 train steps 59280
Length 70
Episode 38743 total samples 486794 train steps 59600
Length 119
Episode 38883 total samples 489301 train steps 59913
Length 16
Episode 39030 total samples 491811 train steps 60227
Length 71
Episode 39163 total samples 494338 train steps 60543
Length 63
Episode 39306 total samples 496843 train steps 60856
Length 37
Episode 39435 total samples 499368 train steps 61171
Length 100
exploration 0.549880300015
Episode 39571 total samples 501870 train steps 61484
Length 24
Episode 39706 total samples 504387 train steps 61799
Length 38
Episode 39848 total samples 506889 train steps 62112
Length 53
Episode 39984 total samples 509394 train steps 62425
Length 23
Episode 40107 total samples 511896 train steps 62737
Length 30
Episode 40240 total samples 514400 train steps 63050
Length 15
Episode 40370 total samples 516920 train steps 63365
Length 60
Episode 40501 total samples 519446 train steps 63681
Length 115
Episode 40633 total samples 521959 train steps 63995
Length 42
Episode 40769 total samples 524498 train steps 64313
Length 108
exploration 0.527373100016
Episode 40904 total samples 527006 train steps 64626
Length 39
Episode 41045 total samples 529517 train steps 64940
Length 39
Episode 41159 total samples 532025 train steps 65254
Length 28
Episode 41284 total samples 534535 train steps 65567
Length 32
Episode 41409 total samples 537042 train steps 65881
Length 36
Episode 41526 total samples 539570 train steps 66197
Length 135
Episode 41652 total samples 542100 train steps 66513
Length 68
Episode 41763 total samples 544607 train steps 66826
Length 37
Episode 41885 total samples 547116 train steps 67140
Length 78
Episode 42002 total samples 549625 train steps 67454
Length 96
exploration 0.504858700016
Episode 42120 total samples 552128 train steps 67766
Length 40
Episode 42239 total samples 554631 train steps 68079
Length 23
Episode 42362 total samples 557140 train steps 68393
Length 25
Episode 42482 total samples 559675 train steps 68710
Length 129
Episode 42602 total samples 562207 train steps 69026
Length 95
Episode 42722 total samples 564719 train steps 69340
Length 63
Episode 42845 total samples 567233 train steps 69655
Length 34
Episode 42961 total samples 569738 train steps 69968
Length 77
Episode 43078 total samples 572254 train steps 70282
Length 132
Episode 43201 total samples 574765 train steps 70596
Length 28
exploration 0.482319100016
Episode 43312 total samples 577290 train steps 70912
Length 66
Episode 43426 total samples 579802 train steps 71226
Length 65
Episode 43558 total samples 582313 train steps 71540
Length 161
Episode 43680 total samples 584815 train steps 71852
Length 17
Episode 43810 total samples 587327 train steps 72166
Length 33
Episode 43933 total samples 589851 train steps 72482
Length 52
Episode 44045 total samples 592369 train steps 72797
Length 132
Episode 44154 total samples 594870 train steps 73109
Length 24
Episode 44294 total samples 597387 train steps 73424
Length 115
Episode 44403 total samples 599924 train steps 73741
Length 88
exploration 0.459811900015
Episode 44527 total samples 602458 train steps 74058
Length 86
Episode 44649 total samples 604965 train steps 74371
Length 14
Episode 44911 total samples 607477 train steps 74685
Length 73
Episode 45022 total samples 609993 train steps 75000
Length 58
Episode 45140 total samples 612498 train steps 75313
Length 11
Episode 45445 total samples 615005 train steps 75626
Length 16
Episode 45793 total samples 617512 train steps 75939
Length 31
Episode 46136 total samples 620031 train steps 76254
Length 42
Episode 46231 total samples 622568 train steps 76571
Length 79
Episode 46332 total samples 625069 train steps 76884
Length 73
exploration 0.437299300015
Episode 46445 total samples 627585 train steps 77199
Length 41
Episode 46564 total samples 630087 train steps 77511
Length 69
Episode 46670 total samples 632600 train steps 77825
Length 54
Episode 46783 total samples 635105 train steps 78139
Length 41
Episode 46891 total samples 637663 train steps 78458
Length 139
Episode 47011 total samples 640172 train steps 78772
Length 35
Episode 47119 total samples 642688 train steps 79086
Length 70
Episode 47237 total samples 645205 train steps 79401
Length 99
Episode 47331 total samples 647712 train steps 79714
Length 18
Episode 47438 total samples 650244 train steps 80031
Length 92
exploration 0.414780400014
Episode 47545 total samples 652771 train steps 80347
Length 98
Episode 47648 total samples 655276 train steps 80660
Length 33
Episode 47756 total samples 657793 train steps 80975
Length 56
Episode 47856 total samples 660294 train steps 81287
Length 38
Episode 47956 total samples 662811 train steps 81602
Length 63
Episode 48055 total samples 665317 train steps 81915
Length 66
Episode 48156 total samples 667867 train steps 82234
Length 152
Episode 48251 total samples 670374 train steps 82547
Length 52
Episode 48374 total samples 672880 train steps 82860
Length 64
exploration 0.392277700013
Episode 48469 total samples 675418 train steps 83178
Length 78
Episode 48565 total samples 677948 train steps 83494
Length 114
Episode 48657 total samples 680504 train steps 83813
Length 118
Episode 48744 total samples 683011 train steps 84127
Length 99
Episode 48832 total samples 685543 train steps 84443
Length 66
Episode 48926 total samples 688057 train steps 84758
Length 132
Episode 49015 total samples 690567 train steps 85071
Length 26
Episode 49109 total samples 693071 train steps 85384
Length 91
Episode 49213 total samples 695599 train steps 85700
Length 133
Episode 49299 total samples 698108 train steps 86014
Length 59
exploration 0.369766900013
Episode 49392 total samples 700645 train steps 86331
Length 76
Episode 49528 total samples 703161 train steps 86646
Length 151
Episode 49623 total samples 705675 train steps 86960
Length 99
Episode 49711 total samples 708217 train steps 87278
Length 106
Episode 49805 total samples 710773 train steps 87597
Length 151
Episode 49895 total samples 713303 train steps 87913
Length 158
Episode 49978 total samples 715806 train steps 88226
Length 15
Episode 50064 total samples 718309 train steps 88539
Length 54
Episode 50150 total samples 720835 train steps 88855
Length 60
Episode 50252 total samples 723400 train steps 89175
Length 164
exploration 0.347262400012
Episode 50337 total samples 725903 train steps 89488
Length 11
Episode 50421 total samples 728426 train steps 89804
Length 56
Episode 50510 total samples 730927 train steps 90116
Length 15
Episode 50601 total samples 733430 train steps 90429
Length 54
Episode 50686 total samples 735953 train steps 90745
Length 49
Episode 50782 total samples 738465 train steps 91059
Length 28
Episode 50865 total samples 740986 train steps 91374
Length 61
Episode 50950 total samples 743499 train steps 91688
Length 36
Episode 51055 total samples 746000 train steps 92000
Length 84
Episode 51140 total samples 748502 train steps 92313
Length 48
exploration 0.324744400011
Episode 51234 total samples 751004 train steps 92626
Length 10
Episode 51317 total samples 753506 train steps 92939
Length 66
Episode 51406 total samples 756039 train steps 93255
Length 66
Episode 51497 total samples 758542 train steps 93568
Length 26
Episode 51583 total samples 761060 train steps 93883
Length 125
Episode 51665 total samples 763566 train steps 94196
Length 98
Episode 51749 total samples 766074 train steps 94510
Length 48
Episode 51831 total samples 768603 train steps 94826
Length 71
Episode 51913 total samples 771136 train steps 95142
Length 83
Episode 51991 total samples 773639 train steps 95455
Length 43
exploration 0.302238100011
Episode 52071 total samples 776170 train steps 95772
Length 108
Episode 52153 total samples 778688 train steps 96086
Length 50
Episode 52235 total samples 781201 train steps 96401
Length 36
Episode 52323 total samples 783717 train steps 96715
Length 56
Episode 52407 total samples 786235 train steps 97030
Length 57
Episode 52486 total samples 788756 train steps 97345
Length 76
Episode 52573 total samples 791273 train steps 97660
Length 123
Episode 52652 total samples 793783 train steps 97973
Length 70
Episode 52731 total samples 796296 train steps 98287
Length 88
Episode 52807 total samples 798804 train steps 98601
Length 80
exploration 0.27971740001
Episode 52886 total samples 801311 train steps 98914
Length 64
Episode 52955 total samples 803837 train steps 99230
Length 122
Episode 53031 total samples 806376 train steps 99547
Length 124
Episode 53114 total samples 808900 train steps 99863
Length 60
Episode 53182 total samples 811430 train steps 100179
Length 63
Episode 53253 total samples 813956 train steps 100495
Length 99
Episode 53324 total samples 816465 train steps 100809
Length 23
Episode 53401 total samples 818980 train steps 101123
Length 79
Episode 53471 total samples 821496 train steps 101437
Length 58
Episode 53542 total samples 824022 train steps 101753
Length 198
exploration 0.257155300009
Episode 53616 total samples 826527 train steps 102066
Length 59
Episode 53692 total samples 829047 train steps 102381
Length 53
Episode 53768 total samples 831579 train steps 102698
Length 134
Episode 53838 total samples 834099 train steps 103013
Length 79
Episode 53908 total samples 836611 train steps 103327
Length 66
Episode 53978 total samples 839124 train steps 103641
Length 130
Episode 54039 total samples 841652 train steps 103957
Length 74
Episode 54119 total samples 844205 train steps 104276
Length 185
Episode 54192 total samples 846709 train steps 104589
Length 71
Episode 54254 total samples 849221 train steps 104903
Length 113
exploration 0.234641800009
Episode 54321 total samples 851723 train steps 105216
Length 106
Episode 54396 total samples 854262 train steps 105533
Length 125
Episode 54457 total samples 856795 train steps 105850
Length 73
Episode 54520 total samples 859328 train steps 106166
Length 70
Episode 54591 total samples 861838 train steps 106480
Length 65
Episode 54657 total samples 864341 train steps 106793
Length 153
Episode 54722 total samples 866884 train steps 107111
Length 110
Episode 54790 total samples 869404 train steps 107426
Length 91
Episode 54866 total samples 871941 train steps 107743
Length 113
Episode 54925 total samples 874444 train steps 108056
Length 69
exploration 0.212102200009
Episode 54995 total samples 877052 train steps 108382
Length 249
Episode 55070 total samples 879559 train steps 108695
Length 76
Episode 55129 total samples 882072 train steps 109009
Length 129
Episode 55192 total samples 884609 train steps 109327
Length 152
Episode 55252 total samples 887115 train steps 109640
Length 78
Episode 55320 total samples 889672 train steps 109959
Length 149
Episode 55376 total samples 892187 train steps 110274
Length 89
Episode 55431 total samples 894690 train steps 110587
Length 78
Episode 55499 total samples 897219 train steps 110903
Length 75
Episode 55557 total samples 899789 train steps 111224
Length 145
exploration 0.189578800009
Episode 55615 total samples 902293 train steps 111537
Length 92
Episode 55672 total samples 904800 train steps 111850
Length 161
Episode 55731 total samples 907326 train steps 112166
Length 84
Episode 55796 total samples 909859 train steps 112483
Length 160
Episode 55857 total samples 912407 train steps 112801
Length 151
Episode 55926 total samples 914966 train steps 113121
Length 187
Episode 55991 total samples 917475 train steps 113435
Length 57
Episode 56050 total samples 919982 train steps 113748
Length 120
Episode 56109 total samples 922497 train steps 114063
Length 96
Episode 56171 total samples 925048 train steps 114381
Length 182
exploration 0.167045500009
Episode 56231 total samples 927566 train steps 114696
Length 55
Episode 56290 total samples 930068 train steps 115009
Length 52
Episode 56342 total samples 932590 train steps 115324
Length 73
Episode 56402 total samples 935117 train steps 115640
Length 81
Episode 56460 total samples 937641 train steps 115956
Length 207
Episode 56521 total samples 940183 train steps 116273
Length 86
Episode 56573 total samples 942736 train steps 116592
Length 106
Episode 56619 total samples 945289 train steps 116912
Length 139
Episode 56674 total samples 947848 train steps 117231
Length 130
Episode 56729 total samples 950377 train steps 117548
Length 103
exploration 0.144408700009
Episode 56782 total samples 952913 train steps 117865
Length 95
Episode 56831 total samples 955433 train steps 118180
Length 74
Episode 56895 total samples 958020 train steps 118503
Length 202
Episode 56946 total samples 960566 train steps 118821
Length 305
Episode 57005 total samples 963079 train steps 119135
Length 74
Episode 57063 total samples 965624 train steps 119453
Length 121
Episode 57106 total samples 968150 train steps 119769
Length 91
Episode 57168 total samples 970671 train steps 120084
Length 286
Episode 57220 total samples 973175 train steps 120397
Length 69
Episode 57271 total samples 975730 train steps 120717
Length 185
exploration 0.12184300001
Episode 57318 total samples 978269 train steps 121034
Length 105
Episode 57376 total samples 980797 train steps 121350
Length 91
Episode 57426 total samples 983336 train steps 121667
Length 114
Episode 57478 total samples 985846 train steps 121981
Length 118
Episode 57527 total samples 988349 train steps 122294
Length 72
Episode 57570 total samples 990898 train steps 122613
Length 97
Episode 57627 total samples 993424 train steps 122928
Length 135
Episode 57665 total samples 995984 train steps 123248
Length 221
Episode 57716 total samples 998523 train steps 123566
Length 105
exploration 0.0999991000096
Episode 57769 total samples 1001049 train steps 123882
Length 111
Episode 57820 total samples 1003560 train steps 124195
Length 55
Episode 57871 total samples 1006121 train steps 124516
Length 161
Episode 57904 total samples 1008652 train steps 124832
Length 73
Episode 57946 total samples 1011185 train steps 125149
Length 102
Episode 57983 total samples 1013766 train steps 125471
Length 182
Episode 58025 total samples 1016369 train steps 125797
Length 282
Episode 58074 total samples 1018933 train steps 126117
Length 238
Episode 58118 total samples 1021471 train steps 126434
Length 108
Episode 58175 total samples 1023980 train steps 126748
Length 93
exploration 0.0999991000096
Episode 58224 total samples 1026486 train steps 127061
Length 295
Episode 58259 total samples 1029047 train steps 127381
Length 122
Episode 58301 total samples 1031560 train steps 127695
Length 76
Episode 58342 total samples 1034095 train steps 128012
Length 162
Episode 58395 total samples 1036596 train steps 128325
Length 39
Episode 58439 total samples 1039113 train steps 128640
Length 48
Episode 58485 total samples 1041628 train steps 128954
Length 74
Episode 58530 total samples 1044159 train steps 129270
Length 278
Episode 58580 total samples 1046671 train steps 129584
Length 160
Episode 58624 total samples 1049283 train steps 129911
Length 252
exploration 0.0999991000096
Episode 58673 total samples 1051843 train steps 130231
Length 182
Episode 58717 total samples 1054364 train steps 130546
Length 91
Episode 58757 total samples 1056917 train steps 130865
Length 111
Episode 58796 total samples 1059448 train steps 131181
Length 118
Episode 58843 total samples 1061998 train steps 131500
Length 100
Episode 58881 total samples 1064519 train steps 131815
Length 107
Episode 58924 total samples 1067040 train steps 132130
Length 205
Episode 58970 total samples 1069555 train steps 132445
Length 70
Episode 59012 total samples 1072072 train steps 132759
Length 128
Episode 59050 total samples 1074586 train steps 133074
Length 63
exploration 0.0999991000096
Episode 59098 total samples 1077092 train steps 133387
Length 111
Episode 59148 total samples 1079645 train steps 133706
Length 173
Episode 59195 total samples 1082174 train steps 134022
Length 130
Episode 59234 total samples 1084744 train steps 134343
Length 258
Episode 59269 total samples 1087259 train steps 134658
Length 99
Episode 59333 total samples 1089764 train steps 134971
Length 145
Episode 59383 total samples 1092350 train steps 135294
Length 200
Episode 59415 total samples 1094953 train steps 135620
Length 216
Episode 59450 total samples 1097454 train steps 135932
Length 60
Episode 59495 total samples 1099974 train steps 136247
Length 164
exploration 0.0999991000096
Episode 59543 total samples 1102489 train steps 136562
Length 54
Episode 59594 total samples 1105030 train steps 136879
Length 154
Episode 59639 total samples 1107591 train steps 137199
Length 212
Episode 59677 total samples 1110110 train steps 137514
Length 109
Episode 59720 total samples 1112627 train steps 137829
Length 272
Episode 59761 total samples 1115224 train steps 138153
Length 269
Episode 59803 total samples 1117747 train steps 138469
Length 166
Episode 59842 total samples 1120256 train steps 138782
Length 50
Episode 59885 total samples 1122777 train steps 139098
Length 188
Episode 59924 total samples 1125501 train steps 139438
Length 479
exploration 0.0999991000096
Episode 59962 total samples 1128018 train steps 139753
Length 58
Episode 60010 total samples 1130598 train steps 140075
Length 164
Episode 60066 total samples 1133105 train steps 140389
Length 23
Episode 60108 total samples 1135637 train steps 140705
Length 76
Episode 60147 total samples 1138164 train steps 141021
Length 287
Episode 60184 total samples 1140695 train steps 141337
Length 123
Episode 60223 total samples 1143206 train steps 141651
Length 89
Episode 60269 total samples 1145733 train steps 141967
Length 90
Episode 60310 total samples 1148270 train steps 142284
Length 134
Episode 60356 total samples 1150839 train steps 142605
Length 297
exploration 0.0999991000096
Episode 60399 total samples 1153361 train steps 142921
Length 129
Episode 60435 total samples 1155889 train steps 143237
Length 336
Episode 60470 total samples 1158404 train steps 143551
Length 77
Episode 60507 total samples 1161008 train steps 143876
Length 345
Episode 60545 total samples 1163527 train steps 144191
Length 131
Episode 60599 total samples 1166292 train steps 144537
Length 682
Episode 60634 total samples 1168858 train steps 144858
Length 162
Episode 60672 total samples 1171484 train steps 145186
Length 347
Episode 60715 total samples 1174084 train steps 145511
Length 348
exploration 0.0999991000096
Episode 60749 total samples 1176661 train steps 145833
Length 335
Episode 60794 total samples 1179168 train steps 146146
Length 54
Episode 60832 total samples 1181695 train steps 146462
Length 131
Episode 60873 total samples 1184306 train steps 146789
Length 398
Episode 60912 total samples 1186918 train steps 147115
Length 368
Episode 60953 total samples 1189431 train steps 147429
Length 174
Episode 61005 total samples 1192049 train steps 147757
Length 309
Episode 61047 total samples 1194574 train steps 148072
Length 54
Episode 61095 total samples 1197081 train steps 148386
Length 57
Episode 61193 total samples 1199612 train steps 148702
Length 71
exploration 0.0999991000096
Episode 61248 total samples 1202130 train steps 149017
Length 41
Episode 61289 total samples 1204657 train steps 149333
Length 165
Episode 61343 total samples 1207231 train steps 149654
Length 151
Episode 61388 total samples 1209748 train steps 149969
Length 83
Episode 61431 total samples 1212281 train steps 150286
Length 92
Episode 61470 total samples 1214788 train steps 150599
Length 274
Episode 61508 total samples 1217315 train steps 150915
Length 77
Episode 61545 total samples 1219862 train steps 151233
Length 159
Episode 61597 total samples 1222414 train steps 151552
Length 106
Episode 61680 total samples 1225025 train steps 151879
Length 249
exploration 0.0999991000096
Episode 61724 total samples 1227529 train steps 152192
Length 126
Episode 61760 total samples 1230053 train steps 152507
Length 263
Episode 61799 total samples 1232562 train steps 152821
Length 115
Episode 61828 total samples 1235082 train steps 153136
Length 84
Episode 61865 total samples 1237602 train steps 153451
Length 63
Episode 61914 total samples 1240143 train steps 153768
Length 169
Episode 61956 total samples 1242835 train steps 154105
Length 424
Episode 61992 total samples 1245405 train steps 154426
Length 139
Episode 62027 total samples 1247998 train steps 154750
Length 284
Episode 62078 total samples 1250556 train steps 155070
Length 115
exploration 0.0999991000096
Episode 62117 total samples 1253099 train steps 155388
Length 221
Episode 62170 total samples 1255614 train steps 155702
Length 82
Episode 62213 total samples 1258123 train steps 156016
Length 247
Episode 62257 total samples 1260625 train steps 156329
Length 129
Episode 62309 total samples 1263151 train steps 156644
Length 60
Episode 62348 total samples 1265654 train steps 156957
Length 30
Episode 62384 total samples 1268227 train steps 157279
Length 241
Episode 62439 total samples 1270754 train steps 157595
Length 286
Episode 62480 total samples 1273289 train steps 157912
Length 123
Episode 62539 total samples 1275887 train steps 158236
Length 354
exploration 0.0999991000096
Episode 62589 total samples 1278457 train steps 158558
Length 140
Episode 62650 total samples 1281066 train steps 158884
Length 292
Episode 62693 total samples 1283584 train steps 159198
Length 69
Episode 62742 total samples 1286100 train steps 159513
Length 85
Episode 62781 total samples 1288635 train steps 159830
Length 160
Episode 62824 total samples 1291149 train steps 160144
Length 145
Episode 62863 total samples 1293653 train steps 160457
Length 106
Episode 62901 total samples 1296160 train steps 160770
Length 63
Episode 62944 total samples 1298695 train steps 161087
Length 82
Episode 62989 total samples 1301220 train steps 161403
Length 49
exploration 0.0999991000096
Episode 63023 total samples 1303730 train steps 161717
Length 301
Episode 63056 total samples 1306251 train steps 162032
Length 55
Episode 63087 total samples 1308776 train steps 162347
Length 264
Episode 63119 total samples 1311288 train steps 162661
Length 50
Episode 63176 total samples 1313828 train steps 162979
Length 239
Episode 63216 total samples 1316334 train steps 163292
Length 224
Episode 63245 total samples 1318849 train steps 163607
Length 517
Episode 63288 total samples 1321373 train steps 163922
Length 114
Episode 63336 total samples 1324001 train steps 164251
Length 269
exploration 0.0999991000096
Episode 63392 total samples 1326539 train steps 164568
Length 110
Episode 63432 total samples 1329109 train steps 164889
Length 206
Episode 63472 total samples 1331657 train steps 165208
Length 120
Episode 63502 total samples 1334253 train steps 165532
Length 332
Episode 63541 total samples 1336777 train steps 165848
Length 54
Episode 63587 total samples 1339283 train steps 166161
Length 272
Episode 63626 total samples 1341785 train steps 166474
Length 73
Episode 63667 total samples 1344380 train steps 166798
Length 235
Episode 63708 total samples 1346923 train steps 167116
Length 177
Episode 63732 total samples 1349451 train steps 167432
Length 225
exploration 0.0999991000096
Episode 63766 total samples 1351953 train steps 167745
Length 53
Episode 63808 total samples 1354470 train steps 168059
Length 227
Episode 63842 total samples 1357047 train steps 168381
Length 244
Episode 63889 total samples 1359588 train steps 168699
Length 157
Episode 63928 total samples 1362102 train steps 169013
Length 50
Episode 63970 total samples 1364606 train steps 169326
Length 11
Episode 64001 total samples 1367147 train steps 169644
Length 384
Episode 64043 total samples 1369652 train steps 169957
Length 83
Episode 64080 total samples 1372218 train steps 170278
Length 141
Episode 64120 total samples 1374726 train steps 170591
Length 187
exploration 0.0999991000096
Episode 64151 total samples 1377228 train steps 170904
Length 86
Episode 64184 total samples 1379778 train steps 171223
Length 112
Episode 64233 total samples 1382319 train steps 171540
Length 148
Episode 64272 total samples 1384883 train steps 171861
Length 222
Episode 64312 total samples 1387460 train steps 172183
Length 172
Episode 64354 total samples 1390034 train steps 172505
Length 324
Episode 64390 total samples 1392549 train steps 172819
Length 92
Episode 64443 total samples 1395166 train steps 173146
Length 264
Episode 64475 total samples 1397690 train steps 173462
Length 102
Episode 64537 total samples 1400225 train steps 173779
Length 169
exploration 0.0999991000096
Episode 64581 total samples 1402758 train steps 174095
Length 174
Episode 64620 total samples 1405287 train steps 174411
Length 390
Episode 64667 total samples 1407788 train steps 174724
Length 67
Episode 64713 total samples 1410301 train steps 175038
Length 93
Episode 64749 total samples 1412845 train steps 175356
Length 153
Episode 64792 total samples 1415394 train steps 175675
Length 234
Episode 64828 total samples 1417921 train steps 175991
Length 57
Episode 64865 total samples 1420424 train steps 176303
Length 38
Episode 64900 total samples 1423080 train steps 176635
Length 365
Episode 64932 total samples 1425613 train steps 176952
Length 292
exploration 0.0999991000096
Episode 64969 total samples 1428123 train steps 177266
Length 83
Episode 65003 total samples 1430736 train steps 177592
Length 250
Episode 65038 total samples 1433257 train steps 177908
Length 212
Episode 65092 total samples 1435829 train steps 178229
Length 157
Episode 65139 total samples 1438360 train steps 178545
Length 61
Episode 65179 total samples 1440877 train steps 178860
Length 79
Episode 65220 total samples 1443379 train steps 179173
Length 107
Episode 65260 total samples 1445939 train steps 179493
Length 361
Episode 65295 total samples 1448539 train steps 179818
Length 215
Episode 65339 total samples 1451069 train steps 180134
Length 130
exploration 0.0999991000096
Episode 65386 total samples 1453605 train steps 180451
Length 107
Episode 65424 total samples 1456238 train steps 180780
Length 278
Episode 65465 total samples 1458741 train steps 181093
Length 42
Episode 65513 total samples 1461255 train steps 181407
Length 269
Episode 65561 total samples 1463771 train steps 181722
Length 47
Episode 65610 total samples 1466280 train steps 182035
Length 113
Episode 65661 total samples 1468822 train steps 182353
Length 174
Episode 65717 total samples 1471430 train steps 182679
Length 276
Episode 65750 total samples 1473963 train steps 182996
Length 96
Episode 65798 total samples 1476487 train steps 183311
Length 152
exploration 0.0999991000096
Episode 65844 total samples 1479033 train steps 183630
Length 268
Episode 65895 total samples 1481534 train steps 183942
Length 19
Episode 65932 total samples 1484067 train steps 184259
Length 81
Episode 65984 total samples 1486675 train steps 184585
Length 231
Episode 66015 total samples 1489203 train steps 184901
Length 67
Episode 66065 total samples 1491844 train steps 185231
Length 335
Episode 66109 total samples 1494365 train steps 185546
Length 104
Episode 66157 total samples 1496883 train steps 185861
Length 47
Episode 66209 total samples 1499434 train steps 186180
Length 345
exploration 0.0999991000096
Episode 66257 total samples 1501954 train steps 186495
Length 160
Episode 66321 total samples 1504487 train steps 186811
Length 65
Episode 66379 total samples 1507003 train steps 187126
Length 59
Episode 66425 total samples 1509583 train steps 187448
Length 199
Episode 66471 total samples 1512184 train steps 187773
Length 455
Episode 66522 total samples 1514729 train steps 188092
Length 120
Episode 66564 total samples 1517317 train steps 188415
Length 233
Episode 66607 total samples 1519895 train steps 188737
Length 180
Episode 66646 total samples 1522434 train steps 189055
Length 135
Episode 66685 total samples 1524948 train steps 189369
Length 194
exploration 0.0999991000096
Episode 66734 total samples 1527457 train steps 189683
Length 279
Episode 66775 total samples 1530049 train steps 190007
Length 183
Episode 66823 total samples 1532583 train steps 190323
Length 90
Episode 66912 total samples 1535087 train steps 190636
Length 101
Episode 66947 total samples 1537601 train steps 190951
Length 168
Episode 66995 total samples 1540120 train steps 191265
Length 184
Episode 67027 total samples 1542653 train steps 191582
Length 145
Episode 67064 total samples 1545181 train steps 191898
Length 514
Episode 67138 total samples 1547707 train steps 192214
Length 121
Episode 67181 total samples 1550257 train steps 192533
Length 106
exploration 0.0999991000096
Episode 67226 total samples 1552764 train steps 192846
Length 68
Episode 67259 total samples 1555274 train steps 193160
Length 517
Episode 67300 total samples 1557778 train steps 193473
Length 67
Episode 67331 total samples 1560336 train steps 193792
Length 141
Episode 67372 total samples 1562873 train steps 194110
Length 80
Episode 67410 total samples 1565427 train steps 194429
Length 438
Episode 67451 total samples 1567971 train steps 194747
Length 123
Episode 67527 total samples 1570547 train steps 195069
Length 161
Episode 67569 total samples 1573159 train steps 195395
Length 234
Episode 67621 total samples 1575679 train steps 195710
Length 164
exploration 0.0999991000096
Episode 67676 total samples 1578274 train steps 196035
Length 267
Episode 67731 total samples 1580795 train steps 196350
Length 102
Episode 67787 total samples 1583437 train steps 196680
Length 391
Episode 67826 total samples 1585961 train steps 196996
Length 54
Episode 67897 total samples 1588559 train steps 197320
Length 238
Episode 67951 total samples 1591185 train steps 197649
Length 729
Episode 67996 total samples 1593705 train steps 197964
Length 143
Episode 68043 total samples 1596222 train steps 198278
Length 126
Episode 68089 total samples 1598728 train steps 198591
Length 128
Episode 68132 total samples 1601244 train steps 198906
Length 165
exploration 0.0999991000096
Episode 68175 total samples 1603813 train steps 199227
Length 177
Episode 68217 total samples 1606371 train steps 199547
Length 152
Episode 68259 total samples 1608987 train steps 199874
Length 581
Episode 68300 total samples 1611571 train steps 200197
Length 208
Episode 68342 total samples 1614128 train steps 200516
Length 134
Episode 68386 total samples 1616671 train steps 200834
Length 97
Episode 68426 total samples 1619206 train steps 201151
Length 88
Episode 68474 total samples 1621728 train steps 201466
Length 248
Episode 68525 total samples 1624321 train steps 201791
Length 229
Episode 68557 total samples 1626846 train steps 202106
Length 295
exploration 0.0999991000096
Episode 68601 total samples 1629367 train steps 202421
Length 88
Episode 68646 total samples 1631897 train steps 202738
Length 67
Episode 68692 total samples 1634465 train steps 203059
Length 153
Episode 68730 total samples 1636972 train steps 203372
Length 26
Episode 68784 total samples 1639545 train steps 203694
Length 301
Episode 68847 total samples 1642080 train steps 204010
Length 73
Episode 68886 total samples 1644611 train steps 204327
Length 67
Episode 68940 total samples 1647187 train steps 204649
Length 186
Episode 68977 total samples 1649692 train steps 204962
Length 125
exploration 0.0999991000096
Episode 69030 total samples 1652261 train steps 205283
Length 288
Episode 69104 total samples 1654858 train steps 205608
Length 226
Episode 69160 total samples 1657396 train steps 205925
Length 141
Episode 69222 total samples 1659985 train steps 206249
Length 213
Episode 69265 total samples 1662493 train steps 206562
Length 171
Episode 69318 total samples 1665016 train steps 206877
Length 148
Episode 69364 total samples 1667581 train steps 207198
Length 162
Episode 69406 total samples 1670169 train steps 207522
Length 204
Episode 69480 total samples 1672714 train steps 207840
Length 127
Episode 69539 total samples 1675221 train steps 208153
Length 50
exploration 0.0999991000096
Episode 69599 total samples 1677746 train steps 208469
Length 149
Episode 69654 total samples 1680321 train steps 208791
Length 207
Episode 69715 total samples 1682836 train steps 209105
Length 247
Episode 69778 total samples 1685414 train steps 209427
Length 231
Episode 69826 total samples 1687922 train steps 209741
Length 56
Episode 69875 total samples 1690434 train steps 210055
Length 223
Episode 69927 total samples 1692936 train steps 210367
Length 47
Episode 69981 total samples 1695462 train steps 210683
Length 185
Episode 70046 total samples 1698013 train steps 211002
Length 110
Episode 70103 total samples 1700540 train steps 211318
Length 73
exploration 0.0999991000096
Episode 70161 total samples 1703135 train steps 211642
Length 227
Episode 70217 total samples 1705647 train steps 211956
Length 56
Episode 70298 total samples 1708164 train steps 212271
Length 245
Episode 70351 total samples 1710724 train steps 212591
Length 137
Episode 70400 total samples 1713283 train steps 212911
Length 208
Episode 70474 total samples 1715873 train steps 213235
Length 446
Episode 70527 total samples 1718383 train steps 213548
Length 95
Episode 70673 total samples 1720888 train steps 213861
Length 32
Episode 70749 total samples 1723394 train steps 214175
Length 33
Episode 70845 total samples 1725901 train steps 214488
Length 118
exploration 0.0999991000096
Episode 70906 total samples 1728449 train steps 214807
Length 210
Episode 70956 total samples 1731049 train steps 215132
Length 296
Episode 70997 total samples 1733582 train steps 215448
Length 126
Episode 71046 total samples 1736133 train steps 215767
Length 126
Episode 71089 total samples 1738648 train steps 216081
Length 99
Episode 71140 total samples 1741166 train steps 216396
Length 44
Episode 71191 total samples 1743691 train steps 216712
Length 96
Episode 71247 total samples 1746204 train steps 217026
Length 70
Episode 71292 total samples 1748843 train steps 217356
Length 295
Episode 71339 total samples 1751355 train steps 217670
Length 96
exploration 0.0999991000096
Episode 71389 total samples 1753878 train steps 217985
Length 124
Episode 71452 total samples 1756404 train steps 218301
Length 120
Episode 71507 total samples 1758909 train steps 218614
Length 103
Episode 71552 total samples 1761479 train steps 218935
Length 157
Episode 71609 total samples 1763998 train steps 219250
Length 183
Episode 71655 total samples 1766507 train steps 219564
Length 114
Episode 71696 total samples 1769011 train steps 219877
Length 35
Episode 71751 total samples 1771518 train steps 220190
Length 76
Episode 71792 total samples 1774111 train steps 220514
Length 386
Episode 71843 total samples 1776663 train steps 220833
Length 124
exploration 0.0999991000096
Episode 71888 total samples 1779181 train steps 221148
Length 58
Episode 71932 total samples 1781713 train steps 221465
Length 89
Episode 71974 total samples 1784218 train steps 221778
Length 205
Episode 72022 total samples 1786741 train steps 222093
Length 134
Episode 72083 total samples 1789300 train steps 222413
Length 130
Episode 72147 total samples 1791803 train steps 222726
Length 58
Episode 72197 total samples 1794311 train steps 223039
Length 35
Episode 72288 total samples 1796816 train steps 223352
Length 32
Episode 72344 total samples 1799333 train steps 223667
Length 164
Episode 72405 total samples 1801839 train steps 223980
Length 276
exploration 0.0999991000096
Episode 72450 total samples 1804362 train steps 224296
Length 119
Episode 72510 total samples 1806870 train steps 224609
Length 17
Episode 72562 total samples 1809394 train steps 224925
Length 66
Episode 72619 total samples 1811910 train steps 225239
Length 386
Episode 72661 total samples 1814416 train steps 225552
Length 151
Episode 72716 total samples 1816946 train steps 225869
Length 335
Episode 72776 total samples 1819460 train steps 226183
Length 30
Episode 72836 total samples 1821964 train steps 226496
Length 89
Episode 72892 total samples 1824658 train steps 226833
Length 410
exploration 0.0999991000096
Episode 72938 total samples 1827180 train steps 227148
Length 218
Episode 73012 total samples 1829698 train steps 227463
Length 108
Episode 73079 total samples 1832242 train steps 227781
Length 143
Episode 73147 total samples 1834745 train steps 228094
Length 107
Episode 73192 total samples 1837298 train steps 228413
Length 209
Episode 73234 total samples 1839896 train steps 228737
Length 197
Episode 73279 total samples 1842426 train steps 229054
Length 271
Episode 73330 total samples 1844993 train steps 229375
Length 188
Episode 73372 total samples 1847494 train steps 229687
Length 64
Episode 73407 total samples 1850063 train steps 230008
Length 251
exploration 0.0999991000096
Episode 73457 total samples 1852567 train steps 230321
Length 20
Episode 73504 total samples 1855125 train steps 230641
Length 120
Episode 73548 total samples 1857661 train steps 230958
Length 152
Episode 73585 total samples 1860174 train steps 231272
Length 212
Episode 73634 total samples 1862679 train steps 231585
Length 103
Episode 73694 total samples 1865203 train steps 231901
Length 82
Episode 73752 total samples 1867740 train steps 232218
Length 90
Episode 73848 total samples 1870253 train steps 232532
Length 103
Episode 73912 total samples 1872811 train steps 232852
Length 124
Episode 73953 total samples 1875313 train steps 233165
Length 31
exploration 0.0999991000096
Episode 74005 total samples 1877835 train steps 233480
Length 56
Episode 74057 total samples 1880346 train steps 233794
Length 120
Episode 74109 total samples 1882905 train steps 234114
Length 129
Episode 74159 total samples 1885421 train steps 234428
Length 96
Episode 74220 total samples 1887999 train steps 234750
Length 229
Episode 74273 total samples 1890517 train steps 235065
Length 235
Episode 74312 total samples 1893029 train steps 235379
Length 219
Episode 74384 total samples 1895586 train steps 235699
Length 169
Episode 74435 total samples 1898089 train steps 236012
Length 166
Episode 74497 total samples 1900628 train steps 236329
Length 129
exploration 0.0999991000096
Episode 74543 total samples 1903148 train steps 236644
Length 252
Episode 74584 total samples 1905660 train steps 236958
Length 153
Episode 74632 total samples 1908174 train steps 237272
Length 80
Episode 74689 total samples 1910688 train steps 237586
Length 61
Episode 74737 total samples 1913225 train steps 237904
Length 74
Episode 74786 total samples 1915773 train steps 238222
Length 97
Episode 74838 total samples 1918277 train steps 238535
Length 108
Episode 74888 total samples 1920780 train steps 238848
Length 125
Episode 74975 total samples 1923298 train steps 239163
Length 92
Episode 75039 total samples 1925842 train steps 239481
Length 97
exploration 0.0999991000096
Episode 75087 total samples 1928372 train steps 239797
Length 236
Episode 75157 total samples 1930877 train steps 240110
Length 166
Episode 75220 total samples 1933402 train steps 240426
Length 104
Episode 75271 total samples 1935961 train steps 240746
Length 118
Episode 75326 total samples 1938525 train steps 241066
Length 134
Episode 75390 total samples 1941028 train steps 241379
Length 116
Episode 75438 total samples 1943604 train steps 241701
Length 246
Episode 75493 total samples 1946201 train steps 242026
Length 197
Episode 75574 total samples 1948741 train steps 242343
Length 128
Episode 75618 total samples 1951254 train steps 242657
Length 205
exploration 0.0999991000096
Episode 75664 total samples 1953778 train steps 242973
Length 112
Episode 75720 total samples 1956286 train steps 243286
Length 55
Episode 75772 total samples 1958794 train steps 243600
Length 169
Episode 75814 total samples 1961325 train steps 243916
Length 65
Episode 75857 total samples 1963938 train steps 244243
Length 229
Episode 75923 total samples 1966485 train steps 244561
Length 98
Episode 76075 total samples 1968986 train steps 244874
Length 10
Episode 76461 total samples 1971488 train steps 245186
Length 39
Episode 76628 total samples 1974009 train steps 245502
Length 43
Episode 76695 total samples 1976550 train steps 245819
Length 115
exploration 0.0999991000096
Episode 76763 total samples 1979103 train steps 246138
Length 113
Episode 76812 total samples 1981663 train steps 246458
Length 121
Episode 76864 total samples 1984207 train steps 246776
Length 102
Episode 76917 total samples 1986708 train steps 247089
Length 78
Episode 76986 total samples 1989262 train steps 247408
Length 137
Episode 77032 total samples 1991784 train steps 247723
Length 46
Episode 77101 total samples 1994300 train steps 248038
Length 51
Episode 77189 total samples 1996870 train steps 248359
Length 288
Episode 77246 total samples 1999403 train steps 248676
Length 236
Episode 77309 total samples 2002048 train steps 249006
Length 290
exploration 0.0999991000096
Episode 77367 total samples 2004581 train steps 249323
Length 67
Episode 77431 total samples 2007085 train steps 249636
Length 163
Episode 77491 total samples 2009587 train steps 249949
Length 20
Episode 77559 total samples 2012121 train steps 250266
Length 168
Episode 77625 total samples 2014652 train steps 250582
Length 89
Episode 77681 total samples 2017160 train steps 250895
Length 100
Episode 77743 total samples 2019694 train steps 251212
Length 120
Episode 77817 total samples 2022227 train steps 251529
Length 84
Episode 77922 total samples 2024778 train steps 251848
Length 190
Episode 77983 total samples 2027281 train steps 252161
Length 48
exploration 0.0999991000096
Episode 78041 total samples 2029799 train steps 252475
Length 124
Episode 78099 total samples 2032350 train steps 252794
Length 156
Episode 78165 total samples 2034862 train steps 253108
Length 81
Episode 78236 total samples 2037387 train steps 253424
Length 59
Episode 78305 total samples 2039937 train steps 253743
Length 153
Episode 78384 total samples 2042459 train steps 254058
Length 138
Episode 78455 total samples 2045042 train steps 254381
Length 239
Episode 78518 total samples 2047554 train steps 254695
Length 69
Episode 78571 total samples 2050089 train steps 255012
Length 125
exploration 0.0999991000096
Episode 78623 total samples 2052705 train steps 255339
Length 306
Episode 78688 total samples 2055219 train steps 255653
Length 163
Episode 78760 total samples 2057723 train steps 255966
Length 97
Episode 78818 total samples 2060287 train steps 256286
Length 138
Episode 78882 total samples 2062800 train steps 256600
Length 52
Episode 78952 total samples 2065331 train steps 256917
Length 62
Episode 79208 total samples 2067842 train steps 257231
Length 212
Episode 79276 total samples 2070366 train steps 257546
Length 141
Episode 79341 total samples 2072896 train steps 257862
Length 88
Episode 79403 total samples 2075460 train steps 258183
Length 156
exploration 0.0999991000096
Episode 79467 total samples 2077964 train steps 258496
Length 11
Episode 79530 total samples 2080478 train steps 258810
Length 98
Episode 79597 total samples 2082985 train steps 259124
Length 76
Episode 79654 total samples 2085503 train steps 259438
Length 54
Episode 79713 total samples 2088026 train steps 259754
Length 89
Episode 79773 total samples 2090594 train steps 260075
Length 162
Episode 79830 total samples 2093109 train steps 260389
Length 29
Episode 79898 total samples 2095628 train steps 260704
Length 131
Episode 79965 total samples 2098155 train steps 261020
Length 124
Episode 80021 total samples 2100736 train steps 261342
Length 371
exploration 0.0999991000096
Episode 80073 total samples 2103248 train steps 261656
Length 82
Episode 80129 total samples 2105779 train steps 261973
Length 82
Episode 80200 total samples 2108286 train steps 262286
Length 55
Episode 80270 total samples 2110787 train steps 262599
Length 65
Episode 80332 total samples 2113310 train steps 262914
Length 83
Episode 80375 total samples 2115839 train steps 263230
Length 68
Episode 80433 total samples 2118365 train steps 263546
Length 100
Episode 80497 total samples 2120871 train steps 263859
Length 55
Episode 80564 total samples 2123390 train steps 264174
Length 52
Episode 80619 total samples 2125914 train steps 264490
Length 52
exploration 0.0999991000096
Episode 80665 total samples 2128429 train steps 264804
Length 285
Episode 80725 total samples 2130946 train steps 265119
Length 196
Episode 80779 total samples 2133498 train steps 265438
Length 495
Episode 80910 total samples 2135999 train steps 265750
Length 9
Episode 81057 total samples 2138516 train steps 266065
Length 51
Episode 81108 total samples 2141116 train steps 266390
Length 384
Episode 81158 total samples 2143624 train steps 266703
Length 63
Episode 81199 total samples 2146137 train steps 267018
Length 94
Episode 81258 total samples 2148711 train steps 267339
Length 187
Episode 81310 total samples 2151289 train steps 267662
Length 293
exploration 0.0999991000096
Episode 81357 total samples 2153817 train steps 267978
Length 75
Episode 81420 total samples 2156330 train steps 268292
Length 160
Episode 81491 total samples 2158859 train steps 268608
Length 59
Episode 81553 total samples 2161390 train steps 268924
Length 149
Episode 81624 total samples 2163898 train steps 269238
Length 36
Episode 81689 total samples 2166445 train steps 269556
Length 107
Episode 81758 total samples 2168964 train steps 269871
Length 75
Episode 81813 total samples 2171515 train steps 270190
Length 165
Episode 81869 total samples 2174019 train steps 270503
Length 219
Episode 81942 total samples 2176593 train steps 270825
Length 181
exploration 0.0999991000096
Episode 81999 total samples 2179115 train steps 271140
Length 186
Episode 82060 total samples 2181653 train steps 271457
Length 193
Episode 82114 total samples 2184164 train steps 271771
Length 266
Episode 82178 total samples 2186669 train steps 272084
Length 29
Episode 82230 total samples 2189172 train steps 272397
Length 113
Episode 82295 total samples 2191688 train steps 272711
Length 218
Episode 82374 total samples 2194196 train steps 273025
Length 159
Episode 82446 total samples 2196710 train steps 273339
Length 70
Episode 82515 total samples 2199315 train steps 273665
Length 318
Episode 82570 total samples 2201833 train steps 273980
Length 100
exploration 0.0999991000096
Episode 82624 total samples 2204386 train steps 274299
Length 111
Episode 82687 total samples 2206902 train steps 274613
Length 46
Episode 82738 total samples 2209409 train steps 274927
Length 17
Episode 82794 total samples 2211977 train steps 275248
Length 197
Episode 82837 total samples 2214647 train steps 275581
Length 343
Episode 82888 total samples 2217201 train steps 275901
Length 129
Episode 82942 total samples 2219758 train steps 276220
Length 122
Episode 82991 total samples 2222276 train steps 276535
Length 113
Episode 83034 total samples 2224811 train steps 276852
Length 117
Episode 83085 total samples 2227321 train steps 277166
Length 48
exploration 0.0999991000096
Episode 83146 total samples 2229824 train steps 277478
Length 157
Episode 83196 total samples 2232331 train steps 277792
Length 47
Episode 83250 total samples 2234848 train steps 278106
Length 65
Episode 83304 total samples 2237393 train steps 278425
Length 124
Episode 83370 total samples 2239903 train steps 278738
Length 27
Episode 83411 total samples 2242420 train steps 279053
Length 159
Episode 83460 total samples 2244938 train steps 279368
Length 53
Episode 83514 total samples 2247497 train steps 279688
Length 202
Episode 83561 total samples 2250011 train steps 280002
Length 31
exploration 0.0999991000096
Episode 83624 total samples 2252538 train steps 280318
Length 58
Episode 83682 total samples 2255046 train steps 280631
Length 51
Episode 83739 total samples 2257556 train steps 280945
Length 173
Episode 83801 total samples 2260081 train steps 281261
Length 209
Episode 83865 total samples 2262603 train steps 281576
Length 71
Episode 83942 total samples 2265128 train steps 281891
Length 161
Episode 84010 total samples 2267633 train steps 282205
Length 65
Episode 84090 total samples 2270157 train steps 282520
Length 64
Episode 84137 total samples 2272667 train steps 282834
Length 47
Episode 84202 total samples 2275196 train steps 283150
Length 64
exploration 0.0999991000096
Episode 84246 total samples 2277711 train steps 283464
Length 92
Episode 84316 total samples 2280222 train steps 283778
Length 101
Episode 84367 total samples 2282723 train steps 284091
Length 39
Episode 84442 total samples 2285243 train steps 284406
Length 50
Episode 84503 total samples 2287748 train steps 284719
Length 248
Episode 84564 total samples 2290261 train steps 285033
Length 219
Episode 84637 total samples 2292794 train steps 285350
Length 145
Episode 84699 total samples 2295349 train steps 285669
Length 185
Episode 84751 total samples 2297854 train steps 285982
Length 53
Episode 84803 total samples 2300466 train steps 286309
Length 250
exploration 0.0999991000096
Episode 84846 total samples 2302976 train steps 286622
Length 258
Episode 84899 total samples 2305494 train steps 286937
Length 115
Episode 84951 total samples 2308066 train steps 287259
Length 181
Episode 85004 total samples 2310597 train steps 287575
Length 76
Episode 85069 total samples 2313110 train steps 287889
Length 37
Episode 85120 total samples 2315642 train steps 288206
Length 139
Episode 85177 total samples 2318146 train steps 288519
Length 108
Episode 85239 total samples 2320678 train steps 288835
Length 75
Episode 85294 total samples 2323186 train steps 289149
Length 133
Episode 85343 total samples 2325797 train steps 289475
Length 330
exploration 0.0999991000096
Episode 85389 total samples 2328299 train steps 289788
Length 42
Episode 85443 total samples 2330828 train steps 290104
Length 68
Episode 85494 total samples 2333363 train steps 290421
Length 173
Episode 85595 total samples 2335879 train steps 290735
Length 49
Episode 85652 total samples 2338396 train steps 291050
Length 140
Episode 85698 total samples 2340986 train steps 291374
Length 194
Episode 85745 total samples 2343490 train steps 291687
Length 141
Episode 85802 total samples 2346010 train steps 292002
Length 105
Episode 85853 total samples 2348531 train steps 292317
Length 281
Episode 85906 total samples 2351036 train steps 292630
Length 184
exploration 0.0999991000096
Episode 85947 total samples 2353558 train steps 292945
Length 258
Episode 85990 total samples 2356115 train steps 293265
Length 141
Episode 86033 total samples 2358622 train steps 293578
Length 208
Episode 86081 total samples 2361204 train steps 293901
Length 252
Episode 86129 total samples 2363720 train steps 294215
Length 182
Episode 86185 total samples 2366225 train steps 294529
Length 127
Episode 86243 total samples 2368731 train steps 294842
Length 80
Episode 86288 total samples 2371277 train steps 295160
Length 228
Episode 86333 total samples 2373813 train steps 295477
Length 367
Episode 86377 total samples 2376333 train steps 295792
Length 125
exploration 0.0999991000096
Episode 86421 total samples 2378929 train steps 296117
Length 192
Episode 86474 total samples 2381508 train steps 296439
Length 211
Episode 86519 total samples 2384016 train steps 296752
Length 81
Episode 86564 total samples 2386519 train steps 297065
Length 196
Episode 86611 total samples 2389097 train steps 297388
Length 217
Episode 86666 total samples 2391602 train steps 297701
Length 426
Episode 86717 total samples 2394222 train steps 298028
Length 305
Episode 86761 total samples 2396828 train steps 298354
Length 254
Episode 86829 total samples 2399330 train steps 298667
Length 14
Episode 86988 total samples 2401857 train steps 298983
Length 373
exploration 0.0999991000096
Episode 87043 total samples 2404447 train steps 299306
Length 309
Episode 87092 total samples 2406961 train steps 299621
Length 160
Episode 87163 total samples 2409477 train steps 299935
Length 171
Episode 87215 total samples 2412001 train steps 300251
Length 61
Episode 87253 total samples 2414609 train steps 300577
Length 318
Episode 87315 total samples 2417158 train steps 300895
Length 175
Episode 87353 total samples 2419685 train steps 301211
Length 76
Episode 87397 total samples 2422221 train steps 301528
Length 73
Episode 87448 total samples 2424764 train steps 301846
Length 131
Episode 87500 total samples 2427340 train steps 302168
Length 202
exploration 0.0999991000096
Episode 87532 total samples 2429967 train steps 302496
Length 291
Episode 87579 total samples 2432474 train steps 302810
Length 29
Episode 87641 total samples 2434978 train steps 303123
Length 55
Episode 87702 total samples 2437591 train steps 303449
Length 231
Episode 87746 total samples 2440092 train steps 303762
Length 138
Episode 87823 total samples 2442634 train steps 304080
Length 87
Episode 87868 total samples 2445155 train steps 304395
Length 164
Episode 87913 total samples 2447684 train steps 304711
Length 77
Episode 87996 total samples 2450204 train steps 305026
Length 82
Episode 88056 total samples 2452706 train steps 305339
Length 108
exploration 0.0999991000096
Episode 88121 total samples 2455240 train steps 305655
Length 94
Episode 88172 total samples 2457834 train steps 305980
Length 189
Episode 88231 total samples 2460402 train steps 306301
Length 140
Episode 88273 total samples 2462923 train steps 306616
Length 144
Episode 88320 total samples 2465501 train steps 306938
Length 230
Episode 88362 total samples 2468039 train steps 307255
Length 115
Episode 88410 total samples 2470568 train steps 307571
Length 135
Episode 88449 total samples 2473107 train steps 307889
Length 116
Episode 88503 total samples 2475620 train steps 308203
Length 81
exploration 0.0999991000096
Episode 88566 total samples 2478139 train steps 308518
Length 112
Episode 88614 total samples 2480710 train steps 308839
Length 455
Episode 88655 total samples 2483219 train steps 309153
Length 114
Episode 88718 total samples 2485720 train steps 309465
Length 48
Episode 88778 total samples 2488250 train steps 309782
Length 184
Episode 88830 total samples 2490773 train steps 310097
Length 94
Episode 88901 total samples 2493290 train steps 310412
Length 82
Episode 88969 total samples 2495820 train steps 310728
Length 66
Episode 89075 total samples 2498321 train steps 311041
Length 50
Episode 89129 total samples 2500978 train steps 311373
Length 444
exploration 0.0999991000096
Episode 89183 total samples 2503514 train steps 311690
Length 118
Episode 89230 total samples 2506018 train steps 312003
Length 130
Episode 89287 total samples 2508531 train steps 312317
Length 45
Episode 89342 total samples 2511034 train steps 312630
Length 53
Episode 89396 total samples 2513543 train steps 312943
Length 221
Episode 89456 total samples 2516053 train steps 313257
Length 167
Episode 89500 total samples 2518596 train steps 313575
Length 237
Episode 89544 total samples 2521111 train steps 313889
Length 177
Episode 89602 total samples 2523612 train steps 314202
Length 84
Episode 89663 total samples 2526204 train steps 314526
Length 302
exploration 0.0999991000096
Episode 89717 total samples 2528729 train steps 314842
Length 169
Episode 89758 total samples 2531248 train steps 315156
Length 126
Episode 89826 total samples 2533769 train steps 315472
Length 165
Episode 89880 total samples 2536520 train steps 315815
Length 516
Episode 89924 total samples 2539088 train steps 316136
Length 200
Episode 89974 total samples 2541601 train steps 316451
Length 43
Episode 90014 total samples 2544139 train steps 316768
Length 136
Episode 90081 total samples 2546690 train steps 317087
Length 148
Episode 90122 total samples 2549264 train steps 317408
Length 287
Episode 90171 total samples 2551843 train steps 317731
Length 174
exploration 0.0999991000096
Episode 90228 total samples 2554349 train steps 318044
Length 33
Episode 90280 total samples 2556859 train steps 318358
Length 19
Episode 90320 total samples 2559374 train steps 318672
Length 47
Episode 90357 total samples 2561901 train steps 318988
Length 76
Episode 90405 total samples 2564416 train steps 319302
Length 264
Episode 90439 total samples 2566918 train steps 319615
Length 34
Episode 90482 total samples 2569490 train steps 319937
Length 173
Episode 90526 total samples 2572014 train steps 320252
Length 93
Episode 90569 total samples 2574530 train steps 320567
Length 172
Episode 90619 total samples 2577034 train steps 320880
Length 53
exploration 0.0999991000096
Episode 90702 total samples 2579606 train steps 321201
Length 192
Episode 90752 total samples 2582132 train steps 321517
Length 80
Episode 90807 total samples 2584645 train steps 321831
Length 70
Episode 90874 total samples 2587151 train steps 322144
Length 25
Episode 90933 total samples 2589665 train steps 322459
Length 60
Episode 90978 total samples 2592222 train steps 322778
Length 391
Episode 91021 total samples 2594725 train steps 323091
Length 134
Episode 91069 total samples 2597245 train steps 323406
Length 124
Episode 91122 total samples 2599750 train steps 323719
Length 48
Episode 91175 total samples 2602251 train steps 324032
Length 120
exploration 0.0999991000096
Episode 91234 total samples 2604801 train steps 324351
Length 224
Episode 91293 total samples 2607317 train steps 324665
Length 42
Episode 91332 total samples 2609852 train steps 324982
Length 103
Episode 91386 total samples 2612428 train steps 325304
Length 164
Episode 91435 total samples 2614962 train steps 325621
Length 274
Episode 91508 total samples 2617483 train steps 325936
Length 109
Episode 91560 total samples 2619985 train steps 326249
Length 95
Episode 91599 total samples 2622541 train steps 326568
Length 131
Episode 91665 total samples 2625052 train steps 326882
Length 38
Episode 91714 total samples 2627557 train steps 327195
Length 21
exploration 0.0999991000096
Episode 91759 total samples 2630280 train steps 327535
Length 568
Episode 91811 total samples 2632887 train steps 327861
Length 312
Episode 91877 total samples 2635438 train steps 328180
Length 142
Episode 91943 total samples 2637962 train steps 328496
Length 48
Episode 92000 total samples 2640481 train steps 328811
Length 49
Episode 92124 total samples 2642984 train steps 329123
Length 14
Episode 92210 total samples 2645498 train steps 329438
Length 77
Episode 92245 total samples 2648011 train steps 329752
Length 150
Episode 92283 total samples 2650559 train steps 330070
Length 144
Episode 92325 total samples 2653066 train steps 330384
Length 381
exploration 0.0999991000096
Episode 92368 total samples 2655606 train steps 330701
Length 146
Episode 92422 total samples 2658136 train steps 331017
Length 100
Episode 92474 total samples 2660820 train steps 331353
Length 408
Episode 92509 total samples 2663354 train steps 331670
Length 148
Episode 92588 total samples 2665864 train steps 331983
Length 61
Episode 92632 total samples 2668376 train steps 332297
Length 31
Episode 92674 total samples 2671206 train steps 332651
Length 726
Episode 92708 total samples 2673729 train steps 332967
Length 181
Episode 92758 total samples 2676363 train steps 333296
Length 364
exploration 0.0999991000096
Episode 92821 total samples 2678889 train steps 333612
Length 159
Episode 92861 total samples 2681394 train steps 333925
Length 48
Episode 92902 total samples 2683930 train steps 334242
Length 103
Episode 92977 total samples 2686497 train steps 334563
Length 186
Episode 93016 total samples 2689003 train steps 334876
Length 147
Episode 93068 total samples 2691512 train steps 335189
Length 119
Episode 93106 total samples 2694030 train steps 335504
Length 102
Episode 93161 total samples 2696625 train steps 335829
Length 277
Episode 93210 total samples 2699149 train steps 336144
Length 69
Episode 93248 total samples 2701664 train steps 336458
Length 131
exploration 0.0999991000096
Episode 93306 total samples 2704216 train steps 336777
Length 446
Episode 93352 total samples 2706769 train steps 337097
Length 218
Episode 93393 total samples 2709322 train steps 337416
Length 165
Episode 93429 total samples 2711825 train steps 337729
Length 269
Episode 93481 total samples 2714365 train steps 338046
Length 99
Episode 93528 total samples 2716974 train steps 338372
Length 257
Episode 93636 total samples 2719490 train steps 338687
Length 139
Episode 93678 total samples 2722010 train steps 339002
Length 63
Episode 93725 total samples 2724537 train steps 339318
Length 60
Episode 93777 total samples 2727086 train steps 339636
Length 153
exploration 0.0999991000096
Episode 93824 total samples 2729594 train steps 339950
Length 625
Episode 93871 total samples 2732103 train steps 340263
Length 67
Episode 93912 total samples 2734648 train steps 340581
Length 237
Episode 93953 total samples 2737188 train steps 340899
Length 112
Episode 93998 total samples 2739737 train steps 341218
Length 145
Episode 94040 total samples 2742303 train steps 341538
Length 170
Episode 94093 total samples 2744902 train steps 341863
Length 307
Episode 94145 total samples 2747556 train steps 342195
Length 345
Episode 94184 total samples 2750161 train steps 342521
Length 237
Episode 94223 total samples 2752682 train steps 342836
Length 131
exploration 0.0999991000096
Episode 94270 total samples 2755185 train steps 343149
Length 190
Episode 94323 total samples 2757710 train steps 343464
Length 90
Episode 94435 total samples 2760275 train steps 343785
Length 153
Episode 94465 total samples 2762854 train steps 344107
Length 270
Episode 94510 total samples 2765361 train steps 344421
Length 52
Episode 94551 total samples 2767893 train steps 344737
Length 152
Episode 94589 total samples 2770528 train steps 345066
Length 336
Episode 94637 total samples 2773122 train steps 345391
Length 333
Episode 94681 total samples 2775641 train steps 345706
Length 65
Episode 94719 total samples 2778166 train steps 346021
Length 179
exploration 0.0999991000096
Episode 94762 total samples 2780705 train steps 346339
Length 191
Episode 94805 total samples 2783250 train steps 346657
Length 98
Episode 94841 total samples 2785752 train steps 346969
Length 221
Episode 94884 total samples 2788289 train steps 347287
Length 130
Episode 94929 total samples 2790810 train steps 347602
Length 150
Episode 94977 total samples 2793311 train steps 347914
Length 96
Episode 95015 total samples 2795892 train steps 348237
Length 582
Episode 95057 total samples 2798429 train steps 348554
Length 145
Episode 95105 total samples 2800980 train steps 348873
Length 240
exploration 0.0999991000096
Episode 95161 total samples 2803491 train steps 349187
Length 90
Episode 95210 total samples 2806008 train steps 349501
Length 182
Episode 95255 total samples 2808520 train steps 349815
Length 103
Episode 95300 total samples 2811140 train steps 350143
Length 296
Episode 95342 total samples 2813680 train steps 350460
Length 134
Episode 95381 total samples 2816207 train steps 350776
Length 293
Episode 95419 total samples 2818764 train steps 351096
Length 271
Episode 95456 total samples 2821267 train steps 351409
Length 240
Episode 95490 total samples 2823854 train steps 351732
Length 201
Episode 95542 total samples 2826416 train steps 352052
Length 195
exploration 0.0999991000096
Episode 95604 total samples 2828946 train steps 352369
Length 136
Episode 95642 total samples 2831506 train steps 352689
Length 185
Episode 95676 total samples 2834115 train steps 353015
Length 259
Episode 95720 total samples 2836665 train steps 353334
Length 102
Episode 95770 total samples 2839196 train steps 353650
Length 87
Episode 95811 total samples 2841771 train steps 353972
Length 185
Episode 95853 total samples 2844297 train steps 354288
Length 51
Episode 95892 total samples 2846899 train steps 354613
Length 265
Episode 95945 total samples 2849409 train steps 354927
Length 33
Episode 95982 total samples 2852063 train steps 355258
Length 310
exploration 0.0999991000096
Episode 96015 total samples 2854631 train steps 355579
Length 192
Episode 96071 total samples 2857182 train steps 355898
Length 152
Episode 96116 total samples 2859829 train steps 356229
Length 370
Episode 96156 total samples 2862388 train steps 356549
Length 189
Episode 96202 total samples 2864913 train steps 356865
Length 74
Episode 96242 total samples 2867460 train steps 357183
Length 139
Episode 96295 total samples 2869987 train steps 357499
Length 55
Episode 96339 total samples 2872548 train steps 357819
Length 181
Episode 96385 total samples 2875071 train steps 358134
Length 292
Episode 96428 total samples 2877615 train steps 358452
Length 137
exploration 0.0999991000096
Episode 96478 total samples 2880148 train steps 358769
Length 133
Episode 96526 total samples 2882657 train steps 359083
Length 97
Episode 96577 total samples 2885213 train steps 359402
Length 155
Episode 96625 total samples 2887727 train steps 359716
Length 49
Episode 96676 total samples 2890300 train steps 360038
Length 193
Episode 96733 total samples 2892824 train steps 360353
Length 132
Episode 96780 total samples 2895362 train steps 360671
Length 169
Episode 96822 total samples 2897865 train steps 360984
Length 147
Episode 96870 total samples 2900522 train steps 361316
Length 362
Episode 96935 total samples 2903085 train steps 361636
Length 146
exploration 0.0999991000096
Episode 96979 total samples 2905602 train steps 361951
Length 74
Episode 97036 total samples 2908141 train steps 362268
Length 98
Episode 97092 total samples 2910701 train steps 362588
Length 327
Episode 97139 total samples 2913237 train steps 362905
Length 141
Episode 97185 total samples 2915756 train steps 363220
Length 304
Episode 97234 total samples 2918270 train steps 363534
Length 44
Episode 97285 total samples 2920772 train steps 363847
Length 85
Episode 97333 total samples 2923320 train steps 364165
Length 130
Episode 97387 total samples 2925824 train steps 364478
Length 175
Episode 97431 total samples 2928346 train steps 364794
Length 69
exploration 0.0999991000096
Episode 97480 total samples 2930968 train steps 365121
Length 377
Episode 97524 total samples 2933500 train steps 365438
Length 98
Episode 97577 total samples 2936021 train steps 365753
Length 65
Episode 97625 total samples 2938525 train steps 366066
Length 97
Episode 97668 total samples 2941062 train steps 366383
Length 147
Episode 97746 total samples 2943563 train steps 366696
Length 19
Episode 97803 total samples 2946080 train steps 367010
Length 52
Episode 97863 total samples 2948701 train steps 367338
Length 319
Episode 97906 total samples 2951263 train steps 367658
Length 560
exploration 0.0999991000096
Episode 97952 total samples 2953781 train steps 367973
Length 51
Episode 97994 total samples 2956322 train steps 368291
Length 124
Episode 98041 total samples 2958881 train steps 368611
Length 167
Episode 98086 total samples 2961456 train steps 368932
Length 166
Episode 98127 total samples 2963984 train steps 369248
Length 157
Episode 98169 total samples 2966498 train steps 369563
Length 40
Episode 98210 total samples 2969020 train steps 369878
Length 60
Episode 98253 total samples 2971524 train steps 370191
Length 117
Episode 98292 total samples 2974159 train steps 370520
Length 325
Episode 98336 total samples 2976744 train steps 370843
Length 194
exploration 0.0999991000096
Episode 98382 total samples 2979268 train steps 371159
Length 51
Episode 98428 total samples 2981823 train steps 371478
Length 129
Episode 98477 total samples 2984432 train steps 371804
Length 357
Episode 98516 total samples 2986964 train steps 372121
Length 88
Episode 98561 total samples 2989515 train steps 372440
Length 238
Episode 98603 total samples 2992023 train steps 372753
Length 83
Episode 98664 total samples 2994558 train steps 373070
Length 421
Episode 98694 total samples 2997078 train steps 373385
Length 209
Episode 98737 total samples 2999580 train steps 373698
Length 74
Episode 98782 total samples 3002125 train steps 374016
Length 131
exploration 0.0999991000096
Episode 98829 total samples 3004678 train steps 374335
Length 128
Episode 98870 total samples 3007466 train steps 374684
Length 608
Episode 98904 total samples 3009978 train steps 374998
Length 83
Episode 98957 total samples 3012520 train steps 375315
Length 93
Episode 99003 total samples 3015072 train steps 375634
Length 191
Episode 99045 total samples 3017721 train steps 375966
Length 550
Episode 99090 total samples 3020245 train steps 376281
Length 147
Episode 99129 total samples 3022746 train steps 376594
Length 150
Episode 99170 total samples 3025260 train steps 376908
Length 166
Episode 99209 total samples 3027766 train steps 377221
Length 259
exploration 0.0999991000096
Episode 99246 total samples 3030380 train steps 377548
Length 421
Episode 99289 total samples 3032885 train steps 377861
Length 214
Episode 99329 total samples 3035422 train steps 378178
Length 83
Episode 99417 total samples 3038029 train steps 378504
Length 492
Episode 99460 total samples 3040554 train steps 378820
Length 201
Episode 99500 total samples 3043173 train steps 379147
Length 292
Episode 99539 total samples 3045713 train steps 379465
Length 79
Episode 99580 total samples 3048282 train steps 379786
Length 367
Episode 99610 total samples 3050792 train steps 380099
Length 109
Episode 99641 total samples 3053412 train steps 380427
Length 592
exploration 0.0999991000096
Episode 99687 total samples 3055927 train steps 380741
Length 36
Episode 99746 total samples 3058444 train steps 381056
Length 373
Episode 99910 total samples 3060952 train steps 381369
Length 16
Episode 100305 total samples 3063459 train steps 381683
Length 23
Episode 100534 total samples 3065982 train steps 381998
Length 53
Episode 100762 total samples 3068499 train steps 382313
Length 34
Episode 100890 total samples 3071008 train steps 382626
Length 38
Episode 101056 total samples 3073524 train steps 382941
Length 33
Episode 101162 total samples 3076047 train steps 383256
Length 52
Episode 101275 total samples 3078566 train steps 383571
Length 63
exploration 0.0999991000096
Episode 101356 total samples 3081071 train steps 383884
Length 54
Episode 101432 total samples 3083601 train steps 384201
Length 97
Episode 101514 total samples 3086114 train steps 384515
Length 50
Episode 101594 total samples 3088633 train steps 384830
Length 38
Episode 101691 total samples 3091167 train steps 385146
Length 137
Episode 101762 total samples 3093671 train steps 385459
Length 63
Episode 101836 total samples 3096172 train steps 385772
Length 46
Episode 101916 total samples 3098678 train steps 386085
Length 39
Episode 101996 total samples 3101216 train steps 386402
Length 90
Episode 102075 total samples 3103730 train steps 386717
Length 46
exploration 0.0999991000096
Episode 102150 total samples 3106244 train steps 387031
Length 92
Episode 102215 total samples 3108770 train steps 387347
Length 61
Episode 102266 total samples 3111324 train steps 387666
Length 239
Episode 102487 total samples 3113868 train steps 387984
Length 100
Episode 102560 total samples 3116378 train steps 388298
Length 56
Episode 102627 total samples 3118894 train steps 388612
Length 138
Episode 102680 total samples 3121468 train steps 388934
Length 362
Episode 102733 total samples 3124005 train steps 389251
Length 191
Episode 102795 total samples 3126520 train steps 389565
Length 65
exploration 0.0999991000096
Episode 102839 total samples 3129031 train steps 389879
Length 173
Episode 102892 total samples 3131538 train steps 390193
Length 67
Episode 102938 total samples 3134084 train steps 390511
Length 118
Episode 102994 total samples 3136601 train steps 390826
Length 84
Episode 103047 total samples 3139190 train steps 391149
Length 179
Episode 103097 total samples 3141788 train steps 391474
Length 195
Episode 103145 total samples 3144310 train steps 391789
Length 156
Episode 103196 total samples 3146832 train steps 392104
Length 73
Episode 103247 total samples 3149333 train steps 392417
Length 98
Episode 103295 total samples 3151993 train steps 392750
Length 343
exploration 0.0999991000096
Episode 103349 total samples 3154498 train steps 393063
Length 67
Episode 103390 total samples 3157110 train steps 393389
Length 407
Episode 103443 total samples 3159670 train steps 393709
Length 249
Episode 103480 total samples 3162187 train steps 394024
Length 74
Episode 103520 total samples 3164776 train steps 394347
Length 231
Episode 103564 total samples 3167301 train steps 394663
Length 92
Episode 103605 total samples 3169960 train steps 394995
Length 331
Episode 103643 total samples 3172540 train steps 395318
Length 231
Episode 103726 total samples 3175155 train steps 395645
Length 337
Episode 103768 total samples 3177729 train steps 395967
Length 235
exploration 0.0999991000096
Episode 103808 total samples 3180303 train steps 396288
Length 452
Episode 103849 total samples 3182848 train steps 396606
Length 153
Episode 103881 total samples 3185422 train steps 396928
Length 571
Episode 103922 total samples 3188072 train steps 397259
Length 432
Episode 103969 total samples 3190639 train steps 397580
Length 207
Episode 104012 total samples 3193183 train steps 397898
Length 97
Episode 104058 total samples 3195705 train steps 398214
Length 64
Episode 104099 total samples 3198268 train steps 398534
Length 138
Episode 104143 total samples 3200785 train steps 398849
Length 183
Episode 104179 total samples 3203287 train steps 399161
Length 49
exploration 0.0999991000096
Episode 104222 total samples 3205796 train steps 399475
Length 154
Episode 104268 total samples 3208300 train steps 399788
Length 48
Episode 104309 total samples 3210835 train steps 400105
Length 107
Episode 104343 total samples 3213432 train steps 400429
Length 222
Episode 104386 total samples 3216239 train steps 400780
Length 663
Episode 104425 total samples 3218809 train steps 401102
Length 179
Episode 104459 total samples 3221411 train steps 401427
Length 236
Episode 104502 total samples 3224011 train steps 401752
Length 278
Episode 104559 total samples 3226555 train steps 402070
Length 146
exploration 0.0999991000096
Episode 104599 total samples 3229105 train steps 402389
Length 158
Episode 104641 total samples 3231626 train steps 402704
Length 318
Episode 104694 total samples 3234200 train steps 403025
Length 215
Episode 104739 total samples 3236802 train steps 403351
Length 208
Episode 104781 total samples 3239324 train steps 403666
Length 120
Episode 104830 total samples 3241896 train steps 403987
Length 214
Episode 104875 total samples 3244437 train steps 404305
Length 87
Episode 104918 total samples 3247032 train steps 404629
Length 260
Episode 104969 total samples 3249537 train steps 404943
Length 139
Episode 105021 total samples 3252093 train steps 405262
Length 336
exploration 0.0999991000096
Episode 105063 total samples 3254619 train steps 405578
Length 268
Episode 105107 total samples 3257132 train steps 405892
Length 45
Episode 105167 total samples 3259658 train steps 406208
Length 265
Episode 105205 total samples 3262196 train steps 406525
Length 134
Episode 105245 total samples 3264721 train steps 406841
Length 116
Episode 105287 total samples 3267291 train steps 407162
Length 312
Episode 105331 total samples 3269813 train steps 407477
Length 172
Episode 105375 total samples 3272332 train steps 407792
Length 270
Episode 105420 total samples 3274928 train steps 408116
Length 195
Episode 105476 total samples 3277442 train steps 408431
Length 28
exploration 0.0999991000096
Episode 105531 total samples 3279974 train steps 408747
Length 133
Episode 105574 total samples 3282520 train steps 409065
Length 287
Episode 105617 total samples 3285035 train steps 409380
Length 49
Episode 105669 total samples 3287675 train steps 409710
Length 285
Episode 105705 total samples 3290184 train steps 410023
Length 63
Episode 105747 total samples 3292884 train steps 410361
Length 512
Episode 105774 total samples 3295541 train steps 410693
Length 332
---------------------------------------------------------------------------
InternalError Traceback (most recent call last)
<ipython-input-4-ee52120e5357> in <module>()
----> 1 car.deepq_learning(num_iter = 1000000, max_steps = 500000, max_learning_steps=1e6)#, reset_replay_memory=True)
/home/ga24pug/DeepRL/q_learning_sarsa/dqn_learning.pyc in deepq_learning(self, num_iter, max_steps, max_learning_steps, learning_rate, reset_replay_memory)
289 done,
290 learning_rate=learning_rate,
--> 291 train_frequency=self.train_frequency)
292 # self.qnn.train_batch()
293 else:
/home/ga24pug/DeepRL/q_learning_sarsa/qnn.pyc in train_batch(self, s, a, r, s_prime, done, a_prime, learning_rate, train_frequency)
607 #regularization
608 if self.reg_weight > 0.0:
--> 609 self.sess.run(self.recons_train_step, feed_dict=feed_dict)
610 else:
611 self.sess.run([self.train_step], feed_dict=feed_dict)
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
370 try:
371 result = self._run(None, fetches, feed_dict, options_ptr,
--> 372 run_metadata_ptr)
373 if run_metadata:
374 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
634 try:
635 results = self._do_run(handle, target_list, unique_fetches,
--> 636 feed_dict_string, options, run_metadata)
637 finally:
638 # The movers are no longer used. Delete them.
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
706 if handle is None:
707 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 708 target_list, options, run_metadata)
709 else:
710 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
726 except KeyError:
727 pass
--> 728 raise type(e)(node_def, op, message)
729
730 def _extend_graph(self):
InternalError: Blas SGEMM launch failed : a.shape=(32, 7744), b.shape=(7744, 512), m=32, n=512, k=7744
[[Node: fc_layer1/Wx_plus_b/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](Reshape, fc_layer1/weights/truediv)]]
Caused by op u'fc_layer1/Wx_plus_b/MatMul', defined at:
File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/ga24pug/mypython-deeprl/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py", line 160, in start
super(ZMQIOLoop, self).start()
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 433, in _handle_events
self._handle_recv()
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 465, in _handle_recv
self._run_callback(callback, msg)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 407, in _run_callback
callback(*args, **kwargs)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-3-3818bc2868df>", line 23, in <module>
pretrain_steps = 0
File "dqn_learning.py", line 161, in __init__
pretrain_steps=pretrain_steps
File "qnn.py", line 433, in __init__
last_hidden = nn_dual_layer_with_decay(conv3_flat, last_conv_layer_output_size, 512, 'fc_layer'+str(1), ema_ops_list, decay=ema_decay_rate)
File "qnn.py", line 266, in nn_dual_layer_with_decay
preactivate_s = tf.matmul(input_tensor_list[0], weights) + biases
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 1209, in matmul
name=name)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1178, in _mat_mul
transpose_b=transpose_b, name=name)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.py", line 704, in apply_op
op_def=op_def)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2260, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1230, in __init__
self._traceback = _extract_stack()
In [ ]:
In [ ]:
In [1]:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
In [2]:
from dqn_learning import *
# import cPickle
# %load_ext autoreload
# %autoreload 2
%pylab inline
# import numpy as np
[2016-08-24 20:02:45,442] Site environment registry incorrect: Scoreboard did not register all envs: set(['AcrobotContinuous-v0'])
Populating the interactive namespace from numpy and matplotlib
WARNING: pylab import has clobbered these variables: ['copy']
`%matplotlib` prevents importing * from pylab and numpy
In [ ]:
car = q_learning(gamma=0.99,
# lambda_=0.9,
# init_epsilon = 0.0,
# update_epsilon = False,
init_epsilon = 1.0,
end_epsilon = 0.1,
update_epsilon = True,
exploration_decrease_length = 1e6,
# N_0= 50,
environment = 'CartPole-v0',
plot_resolution = 30,
nn_size_hidden = [400,300], # [100,100] --> 0.01, [100,100,100] --> similar [400,300]
nn_batch_size = 128,
nn_learning_rate = 2.5e-4,
replay_memory_size = 3e5, # larger than nn_batch_size*1e3 or doesn't train
descent_method = 'rmsprop',
dropout_keep_prob = 0.9,
ema_decay_rate = 0.999,
train_frequency = 4.0,
from_pixels = True,
reg_weight = 0.0,
do_pretrain = False,
pretrain_steps = 0
)
[2016-08-24 20:02:51,584] Making new env: CartPole-v0
conv 128 8 8 2
weights std = 0.125
[8, 8, 2, 32]
bias std = 0.0
conv 512 4 4 32
weights std = 0.0625
[4, 4, 32, 64]
bias std = 0.0
conv 576 3 3 64
weights std = 0.0589255650989
[3, 3, 64, 64]
bias std = 0.0
fc 7744
weights std = 0.0160706086633
bias std = 0.0
fc 512
weights std = 0.0441941738242
bias std = 0.0
Q-learning configured in nn graph
descending with rmsprop
using environment CartPole-v0
qnn target q-learning False False
In [ ]:
car.deepq_learning(num_iter = 1000000, max_steps = 500000, max_learning_steps=1e6)#, reset_replay_memory=True)
Episode 1053 total samples 10035 train steps 1
Length 37
Episode 1316 total samples 12536 train steps 79
Length 15
Episode 1586 total samples 15037 train steps 157
Length 12
Episode 1848 total samples 17540 train steps 236
Length 16
Episode 2106 total samples 20042 train steps 314
Length 11
Episode 2361 total samples 22549 train steps 392
Length 13
exploration 0.977497300001
/usr/lib/pymodules/python2.7/matplotlib/collections.py:548: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
if self._edgecolors == 'face':
Episode 2634 total samples 25056 train steps 470
Length 15
Episode 2903 total samples 27558 train steps 549
Length 9
Episode 3161 total samples 30067 train steps 627
Length 22
Episode 3426 total samples 32574 train steps 705
Length 28
Episode 3686 total samples 35079 train steps 784
Length 13
Episode 3940 total samples 37583 train steps 862
Length 14
Episode 4207 total samples 40091 train steps 940
Length 15
Episode 4465 total samples 42597 train steps 1019
Length 13
Episode 4728 total samples 45099 train steps 1097
Length 30
Episode 4987 total samples 47604 train steps 1175
Length 12
exploration 0.954994600001
Episode 5254 total samples 50110 train steps 1253
Length 15
Episode 5515 total samples 52620 train steps 1332
Length 23
Episode 5778 total samples 55121 train steps 1410
Length 13
Episode 6033 total samples 57630 train steps 1488
Length 24
Episode 6287 total samples 60136 train steps 1567
Length 21
Episode 6554 total samples 62642 train steps 1645
Length 19
Episode 6822 total samples 65144 train steps 1723
Length 38
Episode 7070 total samples 67647 train steps 1801
Length 17
Episode 7338 total samples 70149 train steps 1880
Length 8
Episode 7591 total samples 72659 train steps 1958
Length 56
exploration 0.932483800002
Episode 7847 total samples 75163 train steps 2036
Length 24
Episode 8119 total samples 77671 train steps 2115
Length 15
Episode 8386 total samples 80184 train steps 2193
Length 34
Episode 8654 total samples 82685 train steps 2271
Length 17
Episode 8922 total samples 85186 train steps 2350
Length 15
Episode 9190 total samples 87687 train steps 2428
Length 17
Episode 9461 total samples 90194 train steps 2506
Length 32
Episode 9712 total samples 92695 train steps 2584
Length 9
Episode 9981 total samples 95209 train steps 2663
Length 45
Episode 10234 total samples 97714 train steps 2741
Length 17
exploration 0.909981100003
Episode 10502 total samples 100216 train steps 2819
Length 12
Episode 10767 total samples 102724 train steps 2898
Length 25
Episode 11022 total samples 105235 train steps 2976
Length 26
Episode 11280 total samples 107743 train steps 3054
Length 17
Episode 11529 total samples 110244 train steps 3133
Length 23
Episode 11801 total samples 112749 train steps 3211
Length 23
Episode 12075 total samples 115257 train steps 3289
Length 21
Episode 12340 total samples 117762 train steps 3368
Length 14
Episode 12609 total samples 120264 train steps 3446
Length 15
Episode 12881 total samples 122767 train steps 3524
Length 17
exploration 0.887480200004
Episode 13154 total samples 125272 train steps 3602
Length 18
Episode 13427 total samples 127775 train steps 3680
Length 16
Episode 13691 total samples 130276 train steps 3759
Length 9
Episode 13961 total samples 132782 train steps 3837
Length 20
Episode 14238 total samples 135290 train steps 3915
Length 27
Episode 14502 total samples 137794 train steps 3994
Length 12
Episode 14765 total samples 140295 train steps 4072
Length 12
Episode 15029 total samples 142798 train steps 4150
Length 25
Episode 15278 total samples 145300 train steps 4228
Length 32
Episode 15546 total samples 147801 train steps 4306
Length 17
exploration 0.864979300004
Episode 15830 total samples 150307 train steps 4385
Length 12
Episode 16081 total samples 152812 train steps 4463
Length 27
Episode 16362 total samples 155316 train steps 4541
Length 18
Episode 16625 total samples 157836 train steps 4620
Length 47
Episode 16890 total samples 160348 train steps 4698
Length 26
Episode 17153 total samples 162854 train steps 4777
Length 12
Episode 17421 total samples 165355 train steps 4855
Length 18
Episode 17683 total samples 167867 train steps 4933
Length 29
Episode 17962 total samples 170373 train steps 5012
Length 17
Episode 18241 total samples 172884 train steps 5090
Length 29
exploration 0.842474800005
Episode 18507 total samples 175402 train steps 5169
Length 54
Episode 18774 total samples 177904 train steps 5247
Length 11
Episode 19034 total samples 180408 train steps 5325
Length 20
Episode 19290 total samples 182910 train steps 5403
Length 24
Episode 19555 total samples 185428 train steps 5482
Length 44
Episode 19826 total samples 187937 train steps 5561
Length 51
Episode 20092 total samples 190438 train steps 5639
Length 27
Episode 20361 total samples 192940 train steps 5717
Length 11
Episode 20632 total samples 195442 train steps 5795
Length 18
Episode 20880 total samples 197949 train steps 5873
Length 19
exploration 0.819971200006
Episode 21151 total samples 200450 train steps 5952
Length 13
Episode 21406 total samples 202961 train steps 6030
Length 22
Episode 21660 total samples 205480 train steps 6109
Length 40
Episode 21938 total samples 207985 train steps 6187
Length 21
Episode 22198 total samples 210486 train steps 6265
Length 35
Episode 22462 total samples 212992 train steps 6343
Length 11
Episode 22734 total samples 215495 train steps 6422
Length 32
Episode 22998 total samples 218009 train steps 6500
Length 32
Episode 23268 total samples 220514 train steps 6579
Length 25
Episode 23508 total samples 223016 train steps 6657
Length 16
exploration 0.797469400007
Episode 23767 total samples 225542 train steps 6736
Length 55
Episode 24018 total samples 228046 train steps 6814
Length 26
Episode 24281 total samples 230559 train steps 6892
Length 27
Episode 24536 total samples 233060 train steps 6971
Length 20
Episode 24806 total samples 235561 train steps 7049
Length 35
Episode 25042 total samples 238078 train steps 7127
Length 33
Episode 25290 total samples 240583 train steps 7206
Length 14
Episode 25542 total samples 243084 train steps 7284
Length 13
Episode 25777 total samples 245588 train steps 7362
Length 9
Episode 26012 total samples 248100 train steps 7441
Length 34
exploration 0.774967600007
Episode 26252 total samples 250603 train steps 7519
Length 17
Episode 26496 total samples 253114 train steps 7597
Length 22
Episode 26736 total samples 255619 train steps 7676
Length 26
Episode 26961 total samples 258121 train steps 7754
Length 27
Episode 27196 total samples 260623 train steps 7832
Length 22
Episode 27421 total samples 263130 train steps 7910
Length 43
Episode 27647 total samples 265636 train steps 7989
Length 20
Episode 27860 total samples 268157 train steps 8067
Length 44
Episode 28085 total samples 270673 train steps 8146
Length 36
Episode 28307 total samples 273174 train steps 8224
Length 20
exploration 0.752463100008
Episode 28526 total samples 275678 train steps 8302
Length 9
Episode 28744 total samples 278210 train steps 8382
Length 65
Episode 28958 total samples 280716 train steps 8460
Length 28
Episode 29163 total samples 283218 train steps 8538
Length 16
Episode 29375 total samples 285721 train steps 8616
Length 19
Episode 29583 total samples 288243 train steps 8695
Length 49
Episode 29789 total samples 290747 train steps 8773
Length 24
Episode 30010 total samples 293256 train steps 8852
Length 20
Episode 30202 total samples 295761 train steps 8930
Length 9
Episode 30415 total samples 298273 train steps 9009
Length 41
exploration 0.729949600009
Episode 30626 total samples 300792 train steps 9087
Length 52
Episode 30825 total samples 303297 train steps 9166
Length 12
Episode 31027 total samples 305798 train steps 9244
Length 9
Episode 31227 total samples 308317 train steps 9322
Length 49
Episode 31429 total samples 310829 train steps 9401
Length 51
Episode 31637 total samples 313338 train steps 9479
Length 28
Episode 31843 total samples 315840 train steps 9557
Length 11
Episode 32043 total samples 318350 train steps 9636
Length 57
Episode 32248 total samples 320857 train steps 9714
Length 38
Episode 32447 total samples 323360 train steps 9792
Length 18
exploration 0.70744240001
Episode 32649 total samples 325864 train steps 9871
Length 13
Episode 32856 total samples 328372 train steps 9949
Length 34
Episode 33044 total samples 330883 train steps 10028
Length 33
Episode 33236 total samples 333392 train steps 10106
Length 42
Episode 33428 total samples 335894 train steps 10184
Length 10
Episode 33617 total samples 338403 train steps 10263
Length 20
Episode 33794 total samples 340904 train steps 10341
Length 71
Episode 33984 total samples 343410 train steps 10419
Length 33
Episode 34157 total samples 345918 train steps 10497
Length 26
Episode 34351 total samples 348422 train steps 10576
Length 23
exploration 0.68494060001
Episode 34540 total samples 350938 train steps 10654
Length 80
Episode 34738 total samples 353441 train steps 10733
Length 21
Episode 34925 total samples 355946 train steps 10811
Length 21
Episode 35098 total samples 358449 train steps 10889
Length 44
Episode 35282 total samples 360950 train steps 10967
Length 11
Episode 35466 total samples 363454 train steps 11045
Length 40
Episode 35660 total samples 365958 train steps 11124
Length 29
Episode 35848 total samples 368460 train steps 11202
Length 22
Episode 36031 total samples 370966 train steps 11280
Length 37
Episode 36220 total samples 373482 train steps 11359
Length 34
exploration 0.662428000011
Episode 36383 total samples 375989 train steps 11437
Length 25
Episode 36557 total samples 378506 train steps 11516
Length 64
Episode 36737 total samples 381015 train steps 11594
Length 41
Episode 36924 total samples 383522 train steps 11673
Length 41
Episode 37113 total samples 386053 train steps 11752
Length 65
Episode 37292 total samples 388557 train steps 11830
Length 26
Episode 37479 total samples 391087 train steps 11909
Length 66
Episode 37655 total samples 393589 train steps 11987
Length 26
Episode 37837 total samples 396102 train steps 12066
Length 75
Episode 38029 total samples 398610 train steps 12144
Length 65
exploration 0.639921700012
Episode 38206 total samples 401111 train steps 12222
Length 52
Episode 38389 total samples 403626 train steps 12301
Length 30
Episode 38558 total samples 406131 train steps 12379
Length 14
Episode 38732 total samples 408654 train steps 12458
Length 75
Episode 38895 total samples 411167 train steps 12536
Length 45
Episode 39064 total samples 413672 train steps 12615
Length 23
Episode 39234 total samples 416183 train steps 12693
Length 29
Episode 39395 total samples 418689 train steps 12772
Length 18
Episode 39566 total samples 421191 train steps 12850
Length 33
Episode 39717 total samples 423711 train steps 12928
Length 40
exploration 0.617385700013
Episode 39891 total samples 426222 train steps 13007
Length 30
Episode 40053 total samples 428729 train steps 13085
Length 41
Episode 40227 total samples 431242 train steps 13164
Length 33
Episode 40384 total samples 433753 train steps 13242
Length 52
Episode 40561 total samples 436256 train steps 13320
Length 41
Episode 40729 total samples 438758 train steps 13399
Length 19
Episode 40894 total samples 441266 train steps 13477
Length 50
Episode 41063 total samples 443778 train steps 13556
Length 57
Episode 41224 total samples 446295 train steps 13634
Length 62
Episode 41386 total samples 448826 train steps 13713
Length 87
exploration 0.594864100013
Episode 41537 total samples 451335 train steps 13792
Length 27
Episode 41697 total samples 453839 train steps 13870
Length 17
Episode 41843 total samples 456346 train steps 13948
Length 29
Episode 42001 total samples 458848 train steps 14026
Length 11
Episode 42138 total samples 461359 train steps 14105
Length 23
Episode 42306 total samples 463867 train steps 14183
Length 40
Episode 42472 total samples 466371 train steps 14262
Length 37
Episode 42611 total samples 468872 train steps 14340
Length 23
Episode 42805 total samples 471376 train steps 14418
Length 14
Episode 42955 total samples 473902 train steps 14497
Length 52
exploration 0.572343400014
Episode 43117 total samples 476404 train steps 14575
Length 9
Episode 43264 total samples 478915 train steps 14654
Length 69
Episode 43416 total samples 481416 train steps 14732
Length 15
Episode 43559 total samples 483962 train steps 14811
Length 96
Episode 43713 total samples 486465 train steps 14890
Length 11
Episode 43868 total samples 488979 train steps 14968
Length 78
Episode 44019 total samples 491483 train steps 15046
Length 38
Episode 44156 total samples 493985 train steps 15125
Length 9
Episode 44305 total samples 496489 train steps 15203
Length 27
Episode 44458 total samples 499013 train steps 15282
Length 65
exploration 0.549842500015
Episode 44599 total samples 501520 train steps 15360
Length 43
Episode 44741 total samples 504043 train steps 15439
Length 50
Episode 44872 total samples 506561 train steps 15518
Length 61
Episode 45014 total samples 509064 train steps 15596
Length 94
Episode 45172 total samples 511566 train steps 15674
Length 14
Episode 45313 total samples 514076 train steps 15752
Length 26
Episode 45440 total samples 516583 train steps 15831
Length 39
Episode 45579 total samples 519096 train steps 15909
Length 32
Episode 45712 total samples 521642 train steps 15989
Length 100
Episode 45847 total samples 524154 train steps 16067
Length 106
exploration 0.527337100016
Episode 45976 total samples 526658 train steps 16146
Length 54
Episode 46102 total samples 529179 train steps 16224
Length 56
Episode 46233 total samples 531680 train steps 16302
Length 11
Episode 46373 total samples 534211 train steps 16382
Length 73
Episode 46506 total samples 536737 train steps 16461
Length 51
Episode 46651 total samples 539265 train steps 16540
Length 82
[2016-08-24 21:18:56,532] Observation '[ 2.40469299 3.95600083 -0.24553942 -2.94768578]' is not contained within observation space 'Box(4,)'.
Episode 46778 total samples 541785 train steps 16618
Length 49
Episode 46905 total samples 544288 train steps 16696
Length 15
Episode 47017 total samples 546799 train steps 16775
Length 29
Episode 47142 total samples 549321 train steps 16854
Length 69
exploration 0.504835300016
Episode 47261 total samples 551826 train steps 16932
Length 9
Episode 47393 total samples 554334 train steps 17010
Length 77
Episode 47502 total samples 556835 train steps 17089
Length 16
Episode 47627 total samples 559336 train steps 17167
Length 52
Episode 47732 total samples 561860 train steps 17246
Length 60
Episode 47852 total samples 564361 train steps 17324
Length 12
Episode 47978 total samples 566867 train steps 17402
Length 13
Episode 48106 total samples 569373 train steps 17480
Length 17
Episode 48210 total samples 571890 train steps 17559
Length 48
Episode 48341 total samples 574460 train steps 17639
Length 153
exploration 0.482305600016
Episode 48452 total samples 576994 train steps 17719
Length 79
Episode 48555 total samples 579527 train steps 17798
Length 88
Episode 48663 total samples 582030 train steps 17876
Length 28
Episode 48784 total samples 584543 train steps 17954
Length 65
Episode 48890 total samples 587061 train steps 18033
Length 62
Episode 49000 total samples 589604 train steps 18113
Length 99
Episode 49113 total samples 592112 train steps 18191
Length 64
Episode 49218 total samples 594635 train steps 18270
Length 98
Episode 49339 total samples 597150 train steps 18348
Length 37
Episode 49444 total samples 599662 train steps 18427
Length 62
exploration 0.459782200015
Episode 49534 total samples 602185 train steps 18506
Length 180
Episode 49648 total samples 604694 train steps 18584
Length 25
Episode 49759 total samples 607215 train steps 18663
Length 79
Episode 49869 total samples 609736 train steps 18742
Length 50
Episode 49973 total samples 612260 train steps 18821
Length 79
Episode 50079 total samples 614770 train steps 18899
Length 36
Episode 50189 total samples 617291 train steps 18978
Length 43
Episode 50291 total samples 619825 train steps 19057
Length 67
Episode 50394 total samples 622335 train steps 19135
Length 33
Episode 50478 total samples 624857 train steps 19214
Length 135
exploration 0.437260600015
Episode 50578 total samples 627370 train steps 19293
Length 31
Episode 50689 total samples 629891 train steps 19372
Length 59
Episode 50797 total samples 632468 train steps 19452
Length 198
Episode 50883 total samples 634976 train steps 19530
Length 34
Episode 50983 total samples 637496 train steps 19609
Length 84
Episode 51080 total samples 639997 train steps 19687
Length 17
Episode 51174 total samples 642515 train steps 19766
Length 113
Episode 51277 total samples 645062 train steps 19846
Length 153
Episode 51372 total samples 647564 train steps 19924
Length 116
Episode 51470 total samples 650104 train steps 20003
Length 112
exploration 0.414759700014
Episode 51570 total samples 652618 train steps 20082
Length 28
Episode 51679 total samples 655120 train steps 20160
Length 62
Episode 51781 total samples 657670 train steps 20240
Length 117
Episode 51879 total samples 660211 train steps 20319
Length 90
Episode 51982 total samples 662726 train steps 20398
Length 156
Episode 52077 total samples 665227 train steps 20476
Length 71
Episode 52167 total samples 667729 train steps 20554
Length 14
Episode 52258 total samples 670254 train steps 20633
Length 95
Episode 52355 total samples 672774 train steps 20712
Length 92
exploration 0.392257900013
Episode 52451 total samples 675317 train steps 20791
Length 95
Episode 52545 total samples 677836 train steps 20870
Length 90
Episode 52626 total samples 680398 train steps 20950
Length 155
Episode 52721 total samples 682947 train steps 21030
Length 108
Episode 52806 total samples 685501 train steps 21109
Length 135
Episode 52900 total samples 688014 train steps 21188
Length 41
Episode 52985 total samples 690520 train steps 21266
Length 52
Episode 53072 total samples 693045 train steps 21345
Length 73
Episode 53171 total samples 695554 train steps 21424
Length 63
Episode 53271 total samples 698066 train steps 21502
Length 74
exploration 0.369715600013
Episode 53364 total samples 700576 train steps 21580
Length 74
Episode 53457 total samples 703077 train steps 21659
Length 67
Episode 53534 total samples 705583 train steps 21737
Length 37
Episode 53625 total samples 708099 train steps 21816
Length 42
Episode 53707 total samples 710605 train steps 21894
Length 22
Episode 53786 total samples 713144 train steps 21973
Length 95
Episode 53874 total samples 715649 train steps 22052
Length 21
Episode 53953 total samples 718198 train steps 22131
Length 140
Episode 54036 total samples 720711 train steps 22210
Length 68
Episode 54118 total samples 723282 train steps 22290
Length 216
exploration 0.347208400012
Episode 54195 total samples 725810 train steps 22369
Length 56
Episode 54271 total samples 728315 train steps 22447
Length 35
Episode 54358 total samples 730848 train steps 22526
Length 99
Episode 54439 total samples 733375 train steps 22605
Length 64
Episode 54508 total samples 735900 train steps 22684
Length 115
Episode 54589 total samples 738422 train steps 22763
Length 121
Episode 54676 total samples 740931 train steps 22842
Length 18
Episode 54765 total samples 743446 train steps 22920
Length 111
Episode 54845 total samples 745947 train steps 22998
Length 30
Episode 54929 total samples 748472 train steps 23077
Length 131
exploration 0.324683200011
Episode 55004 total samples 751008 train steps 23156
Length 91
Episode 55082 total samples 753518 train steps 23235
Length 134
Episode 55146 total samples 756062 train steps 23314
Length 104
Episode 55210 total samples 758584 train steps 23393
Length 48
Episode 55276 total samples 761085 train steps 23471
Length 15
Episode 55358 total samples 763595 train steps 23550
Length 130
Episode 55431 total samples 766114 train steps 23629
Length 51
Episode 55526 total samples 768626 train steps 23707
Length 92
Episode 55595 total samples 771153 train steps 23786
Length 68
Episode 55667 total samples 773729 train steps 23867
Length 175
exploration 0.302176000011
Episode 55750 total samples 776236 train steps 23945
Length 61
Episode 55811 total samples 778760 train steps 24024
Length 57
Episode 55886 total samples 781282 train steps 24103
Length 53
Episode 55972 total samples 783795 train steps 24181
Length 61
Episode 56038 total samples 786306 train steps 24260
Length 88
Episode 56102 total samples 788863 train steps 24339
Length 229
Episode 56172 total samples 791392 train steps 24418
Length 87
Episode 56249 total samples 793950 train steps 24498
Length 117
Episode 56311 total samples 796487 train steps 24578
Length 101
Episode 56377 total samples 799002 train steps 24656
Length 159
exploration 0.27963640001
Episode 56436 total samples 801513 train steps 24735
Length 53
Episode 56512 total samples 804053 train steps 24814
Length 129
Episode 56576 total samples 806570 train steps 24893
Length 165
Episode 56630 total samples 809098 train steps 24972
Length 125
Episode 56683 total samples 811623 train steps 25051
Length 187
Episode 56754 total samples 814138 train steps 25129
Length 126
Episode 56819 total samples 816643 train steps 25208
Length 47
Episode 56884 total samples 819167 train steps 25286
Length 47
Episode 56937 total samples 821717 train steps 25366
Length 119
Episode 56993 total samples 824220 train steps 25444
Length 77
exploration 0.257113900009
Episode 57060 total samples 826768 train steps 25524
Length 158
Episode 57129 total samples 829280 train steps 25602
Length 30
Episode 57206 total samples 831782 train steps 25681
Length 168
Episode 57277 total samples 834289 train steps 25759
Length 24
Episode 57330 total samples 836848 train steps 25839
Length 148
Episode 57408 total samples 839382 train steps 25918
Length 87
Episode 57473 total samples 841883 train steps 25996
Length 164
Episode 57531 total samples 844446 train steps 26076
Length 157
Episode 57588 total samples 847030 train steps 26157
Length 199
Episode 57651 total samples 849531 train steps 26235
Length 127
exploration 0.234592300009
Episode 57701 total samples 852045 train steps 26314
Length 96
Episode 57757 total samples 854556 train steps 26392
Length 70
Episode 57824 total samples 857075 train steps 26471
Length 78
Episode 57872 total samples 859628 train steps 26551
Length 134
Episode 57932 total samples 862166 train steps 26630
Length 82
Episode 57989 total samples 864704 train steps 26709
Length 163
Episode 58040 total samples 867226 train steps 26788
Length 52
Episode 58096 total samples 869732 train steps 26867
Length 11
Episode 58147 total samples 872283 train steps 26946
Length 184
Episode 58204 total samples 874790 train steps 27025
Length 49
exploration 0.212075200009
Episode 58282 total samples 877311 train steps 27103
Length 70
Episode 58334 total samples 879830 train steps 27182
Length 128
Episode 58384 total samples 882339 train steps 27261
Length 25
Episode 58439 total samples 884861 train steps 27339
Length 174
Episode 58508 total samples 887397 train steps 27419
Length 71
Episode 58570 total samples 889946 train steps 27498
Length 157
Episode 58624 total samples 892464 train steps 27577
Length 56
Episode 58681 total samples 895020 train steps 27657
Length 225
Episode 58731 total samples 897534 train steps 27735
Length 187
Episode 58783 total samples 900062 train steps 27814
Length 159
exploration 0.189496000009
Episode 58837 total samples 902619 train steps 27894
Length 115
Episode 58894 total samples 905153 train steps 27974
Length 162
Episode 58942 total samples 907685 train steps 28053
Length 134
Episode 58989 total samples 910244 train steps 28133
Length 235
Episode 59043 total samples 912762 train steps 28211
Length 115
Episode 59089 total samples 915277 train steps 28290
Length 69
Episode 59145 total samples 917785 train steps 28368
Length 33
Episode 59202 total samples 920322 train steps 28448
Length 107
Episode 59253 total samples 922898 train steps 28528
Length 176
Episode 59310 total samples 925429 train steps 28607
Length 163
exploration 0.166995100009
Episode 59364 total samples 927950 train steps 28686
Length 75
Episode 59405 total samples 930495 train steps 28765
Length 93
Episode 59444 total samples 933006 train steps 28844
Length 52
Episode 59489 total samples 935515 train steps 28922
Length 131
Episode 59537 total samples 938051 train steps 29002
Length 165
Episode 59591 total samples 940565 train steps 29080
Length 105
Episode 59650 total samples 943071 train steps 29158
Length 219
Episode 59701 total samples 945707 train steps 29241
Length 336
Episode 59753 total samples 948212 train steps 29319
Length 16
exploration 0.144450100009
Episode 59799 total samples 950717 train steps 29397
Length 149
Episode 59838 total samples 953269 train steps 29477
Length 133
Episode 59878 total samples 955806 train steps 29556
Length 117
Episode 59928 total samples 958382 train steps 29637
Length 254
Episode 59958 total samples 960910 train steps 29716
Length 163
Episode 59997 total samples 963415 train steps 29794
Length 149
Episode 60036 total samples 965922 train steps 29873
Length 36
Episode 60070 total samples 968452 train steps 29952
Length 281
Episode 60107 total samples 970995 train steps 30031
Length 191
Episode 60151 total samples 973545 train steps 30111
Length 214
exploration 0.12194920001
Episode 60199 total samples 976099 train steps 30191
Length 111
Episode 60248 total samples 978607 train steps 30269
Length 138
Episode 60292 total samples 981108 train steps 30347
Length 21
Episode 60336 total samples 983615 train steps 30425
Length 195
Episode 60381 total samples 986122 train steps 30504
Length 71
Episode 60420 total samples 988703 train steps 30584
Length 207
Episode 60458 total samples 991246 train steps 30664
Length 111
Episode 60499 total samples 993753 train steps 30742
Length 122
Episode 60531 total samples 996335 train steps 30823
Length 246
Episode 60571 total samples 998860 train steps 30902
Length 67
exploration 0.0999991000096
Episode 60603 total samples 1001383 train steps 30981
Length 71
Episode 60642 total samples 1003921 train steps 31060
Length 178
Episode 60681 total samples 1006435 train steps 31139
Length 31
Episode 60736 total samples 1009005 train steps 31219
Length 209
Episode 60782 total samples 1011560 train steps 31299
Length 168
Episode 60820 total samples 1014084 train steps 31378
Length 88
Episode 60872 total samples 1016614 train steps 31457
Length 86
Episode 60917 total samples 1019138 train steps 31536
Length 113
Episode 60959 total samples 1021674 train steps 31615
Length 96
Episode 60993 total samples 1024206 train steps 31694
Length 116
exploration 0.0999991000096
Episode 61031 total samples 1026708 train steps 31772
Length 46
Episode 61070 total samples 1029322 train steps 31854
Length 237
Episode 61117 total samples 1031840 train steps 31932
Length 110
Episode 61155 total samples 1034354 train steps 32011
Length 196
Episode 61195 total samples 1036959 train steps 32092
Length 547
Episode 61234 total samples 1039574 train steps 32174
Length 298
Episode 61278 total samples 1042087 train steps 32253
Length 109
Episode 61312 total samples 1044696 train steps 32334
Length 220
Episode 61339 total samples 1047212 train steps 32413
Length 89
Episode 61382 total samples 1049800 train steps 32494
Length 178
exploration 0.0999991000096
Episode 61427 total samples 1052324 train steps 32573
Length 205
Episode 61471 total samples 1054869 train steps 32652
Length 119
Episode 61502 total samples 1057396 train steps 32731
Length 92
Episode 61543 total samples 1059912 train steps 32810
Length 447
Episode 61576 total samples 1062463 train steps 32889
Length 271
Episode 61630 total samples 1065019 train steps 32969
Length 114
Episode 61672 total samples 1067554 train steps 33049
Length 102
Episode 61714 total samples 1070098 train steps 33128
Length 137
Episode 61757 total samples 1072644 train steps 33208
Length 147
Episode 61803 total samples 1075153 train steps 33286
Length 158
exploration 0.0999991000096
Episode 61844 total samples 1077699 train steps 33366
Length 160
Episode 61881 total samples 1080257 train steps 33446
Length 118
Episode 61922 total samples 1082826 train steps 33526
Length 215
Episode 61959 total samples 1085452 train steps 33608
Length 373
Episode 61998 total samples 1088007 train steps 33688
Length 116
Episode 62047 total samples 1090547 train steps 33767
Length 188
Episode 62083 total samples 1093056 train steps 33845
Length 46
Episode 62153 total samples 1095676 train steps 33927
Length 279
Episode 62198 total samples 1098178 train steps 34006
Length 82
Episode 62228 total samples 1100707 train steps 34085
Length 194
exploration 0.0999991000096
Episode 62274 total samples 1103232 train steps 34163
Length 133
Episode 62314 total samples 1105755 train steps 34242
Length 185
Episode 62351 total samples 1108386 train steps 34325
Length 310
Episode 62398 total samples 1110920 train steps 34404
Length 130
Episode 62435 total samples 1113553 train steps 34486
Length 297
Episode 62485 total samples 1116089 train steps 34565
Length 126
Episode 62521 total samples 1118625 train steps 34645
Length 140
Episode 62564 total samples 1121160 train steps 34724
Length 72
Episode 62614 total samples 1123713 train steps 34804
Length 153
exploration 0.0999991000096
Episode 62661 total samples 1126232 train steps 34882
Length 127
Episode 62703 total samples 1128780 train steps 34962
Length 130
Episode 62742 total samples 1131351 train steps 35042
Length 200
Episode 62786 total samples 1133865 train steps 35121
Length 60
Episode 62822 total samples 1136366 train steps 35199
Length 113
Episode 62862 total samples 1138915 train steps 35279
Length 118
Episode 62905 total samples 1141455 train steps 35358
Length 127
Episode 62932 total samples 1143969 train steps 35437
Length 147
Episode 62969 total samples 1146495 train steps 35515
Length 320
Episode 63006 total samples 1149024 train steps 35594
Length 115
exploration 0.0999991000096
Episode 63037 total samples 1151575 train steps 35674
Length 261
Episode 63078 total samples 1154128 train steps 35754
Length 122
Episode 63113 total samples 1156629 train steps 35832
Length 78
Episode 63153 total samples 1159132 train steps 35910
Length 100
Episode 63192 total samples 1161636 train steps 35989
Length 92
Episode 63240 total samples 1164209 train steps 36069
Length 222
Episode 63275 total samples 1166739 train steps 36148
Length 210
Episode 63308 total samples 1169268 train steps 36227
Length 175
Episode 63363 total samples 1171783 train steps 36306
Length 112
Episode 63416 total samples 1174413 train steps 36388
Length 259
exploration 0.0999991000096
Episode 63462 total samples 1176947 train steps 36467
Length 222
Episode 63497 total samples 1179549 train steps 36548
Length 297
Episode 63539 total samples 1182060 train steps 36627
Length 74
Episode 63574 total samples 1184633 train steps 36707
Length 168
Episode 63629 total samples 1187196 train steps 36787
Length 140
Episode 63690 total samples 1189703 train steps 36866
Length 117
Episode 63742 total samples 1192223 train steps 36944
Length 365
Episode 63797 total samples 1194733 train steps 37023
Length 187
Episode 63837 total samples 1197258 train steps 37102
Length 126
Episode 63872 total samples 1199846 train steps 37183
Length 288
exploration 0.0999991000096
Episode 63917 total samples 1202377 train steps 37262
Length 155
Episode 63958 total samples 1204975 train steps 37343
Length 216
Episode 63990 total samples 1207488 train steps 37421
Length 72
Episode 64024 total samples 1209992 train steps 37500
Length 138
Episode 64055 total samples 1212612 train steps 37582
Length 324
Episode 64091 total samples 1215151 train steps 37661
Length 168
Episode 64125 total samples 1217808 train steps 37744
Length 379
Episode 64192 total samples 1220341 train steps 37823
Length 213
Episode 64244 total samples 1222900 train steps 37903
Length 158
Episode 64281 total samples 1225410 train steps 37982
Length 95
exploration 0.0999991000096
Episode 64324 total samples 1227932 train steps 38060
Length 109
Episode 64373 total samples 1230479 train steps 38140
Length 96
Episode 64419 total samples 1232996 train steps 38219
Length 151
Episode 64460 total samples 1235535 train steps 38298
Length 241
Episode 64502 total samples 1238067 train steps 38377
Length 92
Episode 64545 total samples 1240607 train steps 38456
Length 150
Episode 64603 total samples 1243166 train steps 38536
Length 125
Episode 64648 total samples 1245709 train steps 38616
Length 119
Episode 64693 total samples 1248223 train steps 38694
Length 82
Episode 64736 total samples 1250781 train steps 38774
Length 152
exploration 0.0999991000096
Episode 64778 total samples 1253342 train steps 38854
Length 156
Episode 64815 total samples 1255914 train steps 38935
Length 237
Episode 64865 total samples 1258456 train steps 39014
Length 230
Episode 64901 total samples 1261069 train steps 39096
Length 299
Episode 64938 total samples 1263663 train steps 39177
Length 231
Episode 64977 total samples 1266266 train steps 39258
Length 368
Episode 65019 total samples 1268794 train steps 39337
Length 200
Episode 65071 total samples 1271313 train steps 39416
Length 131
Episode 65105 total samples 1273856 train steps 39495
Length 148
exploration 0.0999991000096
Episode 65132 total samples 1276448 train steps 39576
Length 332
Episode 65167 total samples 1279016 train steps 39657
Length 162
Episode 65203 total samples 1281558 train steps 39736
Length 116
Episode 65245 total samples 1284243 train steps 39820
Length 414
Episode 65295 total samples 1286824 train steps 39901
Length 161
Episode 65339 total samples 1289422 train steps 39982
Length 361
Episode 65380 total samples 1291927 train steps 40060
Length 111
Episode 65420 total samples 1294432 train steps 40138
Length 104
Episode 65465 total samples 1296994 train steps 40219
Length 166
Episode 65503 total samples 1299551 train steps 40298
Length 209
exploration 0.0999991000096
Episode 65546 total samples 1302069 train steps 40377
Length 149
Episode 65600 total samples 1304619 train steps 40457
Length 155
Episode 65650 total samples 1307146 train steps 40536
Length 99
Episode 65688 total samples 1309690 train steps 40615
Length 175
Episode 65721 total samples 1312247 train steps 40695
Length 138
Episode 65769 total samples 1314812 train steps 40775
Length 202
Episode 65812 total samples 1317317 train steps 40854
Length 80
Episode 65862 total samples 1319825 train steps 40932
Length 92
Episode 65915 total samples 1322338 train steps 41011
Length 140
Episode 65956 total samples 1324874 train steps 41090
Length 263
exploration 0.0999991000096
Episode 65984 total samples 1327389 train steps 41168
Length 64
Episode 66040 total samples 1329935 train steps 41248
Length 116
Episode 66078 total samples 1332529 train steps 41329
Length 476
Episode 66117 total samples 1335090 train steps 41409
Length 202
Episode 66173 total samples 1337623 train steps 41488
Length 171
Episode 66209 total samples 1340152 train steps 41567
Length 71
Episode 66236 total samples 1342703 train steps 41647
Length 110
Episode 66280 total samples 1345228 train steps 41726
Length 196
Episode 66331 total samples 1347772 train steps 41805
Length 183
Episode 66377 total samples 1350328 train steps 41885
Length 169
exploration 0.0999991000096
Episode 66426 total samples 1352913 train steps 41966
Length 194
Episode 66472 total samples 1355427 train steps 42045
Length 40
Episode 66524 total samples 1357933 train steps 42123
Length 224
Episode 66565 total samples 1360474 train steps 42202
Length 252
Episode 66595 total samples 1362980 train steps 42281
Length 34
Episode 66624 total samples 1365494 train steps 42359
Length 52
Episode 66662 total samples 1368084 train steps 42440
Length 365
Episode 66698 total samples 1370660 train steps 42521
Length 442
Episode 66744 total samples 1373165 train steps 42599
Length 13
Episode 66802 total samples 1375713 train steps 42679
Length 113
exploration 0.0999991000096
Episode 66839 total samples 1378234 train steps 42757
Length 46
Episode 66872 total samples 1380815 train steps 42838
Length 243
Episode 66913 total samples 1383365 train steps 42918
Length 100
Episode 66953 total samples 1385869 train steps 42996
Length 231
Episode 66993 total samples 1388436 train steps 43076
Length 378
Episode 67058 total samples 1390955 train steps 43155
Length 86
Episode 67096 total samples 1393490 train steps 43234
Length 132
Episode 67132 total samples 1396011 train steps 43313
Length 193
Episode 67164 total samples 1398538 train steps 43392
Length 317
Episode 67206 total samples 1401220 train steps 43476
Length 367
exploration 0.0999991000096
Episode 67257 total samples 1403755 train steps 43555
Length 149
Episode 67310 total samples 1406327 train steps 43635
Length 164
Episode 67347 total samples 1408835 train steps 43714
Length 147
Episode 67400 total samples 1411354 train steps 43792
Length 186
Episode 67451 total samples 1413857 train steps 43871
Length 47
Episode 67494 total samples 1416413 train steps 43950
Length 321
Episode 67543 total samples 1418915 train steps 44029
Length 14
Episode 67824 total samples 1421430 train steps 44107
Length 33
Episode 67986 total samples 1423948 train steps 44186
Length 35
Episode 68024 total samples 1426455 train steps 44264
Length 146
exploration 0.0999991000096
Episode 68067 total samples 1428969 train steps 44343
Length 106
Episode 68102 total samples 1431540 train steps 44423
Length 350
Episode 68145 total samples 1434059 train steps 44502
Length 96
Episode 68182 total samples 1436560 train steps 44580
Length 283
Episode 68221 total samples 1439109 train steps 44660
Length 123
Episode 68263 total samples 1441626 train steps 44738
Length 89
Episode 68300 total samples 1444244 train steps 44820
Length 359
Episode 68357 total samples 1446789 train steps 44900
Length 178
Episode 68401 total samples 1449386 train steps 44981
Length 223
exploration 0.0999991000096
Episode 68449 total samples 1451897 train steps 45059
Length 122
Episode 68513 total samples 1454418 train steps 45138
Length 170
Episode 68556 total samples 1456955 train steps 45217
Length 118
Episode 68666 total samples 1459489 train steps 45297
Length 99
Episode 68724 total samples 1462024 train steps 45376
Length 92
Episode 68765 total samples 1464544 train steps 45454
Length 111
Episode 68823 total samples 1467054 train steps 45533
Length 21
Episode 68881 total samples 1469591 train steps 45612
Length 269
Episode 68934 total samples 1472092 train steps 45690
Length 60
Episode 68981 total samples 1474613 train steps 45769
Length 88
exploration 0.0999991000096
Episode 69045 total samples 1477184 train steps 45849
Length 147
Episode 69093 total samples 1479792 train steps 45931
Length 236
Episode 69139 total samples 1482309 train steps 46010
Length 270
Episode 69177 total samples 1484847 train steps 46089
Length 96
Episode 69241 total samples 1487376 train steps 46168
Length 62
Episode 69298 total samples 1489895 train steps 46247
Length 82
Episode 69354 total samples 1492451 train steps 46327
Length 145
Episode 69398 total samples 1494975 train steps 46405
Length 71
Episode 69452 total samples 1497511 train steps 46485
Length 131
Episode 69500 total samples 1500143 train steps 46567
Length 370
exploration 0.0999991000096
Episode 69538 total samples 1502735 train steps 46648
Length 238
Episode 69579 total samples 1505311 train steps 46728
Length 260
Episode 69637 total samples 1507836 train steps 46807
Length 137
Episode 69677 total samples 1510394 train steps 46887
Length 158
Episode 69713 total samples 1512950 train steps 46967
Length 233
Episode 69761 total samples 1515479 train steps 47046
Length 93
Episode 69805 total samples 1517997 train steps 47125
Length 129
Episode 69845 total samples 1520611 train steps 47207
Length 261
Episode 69890 total samples 1523166 train steps 47286
Length 111
Episode 69932 total samples 1525733 train steps 47367
Length 157
exploration 0.0999991000096
Episode 69967 total samples 1528273 train steps 47446
Length 152
Episode 70000 total samples 1530807 train steps 47525
Length 79
Episode 70034 total samples 1533369 train steps 47605
Length 279
Episode 70081 total samples 1535911 train steps 47685
Length 100
Episode 70123 total samples 1538449 train steps 47764
Length 136
Episode 70164 total samples 1540960 train steps 47842
Length 47
Episode 70207 total samples 1543501 train steps 47922
Length 222
Episode 70243 total samples 1546053 train steps 48002
Length 108
Episode 70293 total samples 1548559 train steps 48080
Length 72
Episode 70341 total samples 1551081 train steps 48159
Length 116
exploration 0.0999991000096
Episode 70403 total samples 1553605 train steps 48238
Length 90
Episode 70444 total samples 1556120 train steps 48316
Length 43
Episode 70475 total samples 1558653 train steps 48395
Length 89
Episode 70516 total samples 1561194 train steps 48475
Length 139
Episode 70568 total samples 1563706 train steps 48553
Length 62
Episode 70645 total samples 1566207 train steps 48631
Length 10
Episode 70879 total samples 1568710 train steps 48710
Length 33
Episode 70968 total samples 1571310 train steps 48791
Length 316
Episode 71016 total samples 1573844 train steps 48870
Length 185
Episode 71058 total samples 1576401 train steps 48950
Length 120
exploration 0.0999991000096
Episode 71095 total samples 1578927 train steps 49029
Length 288
Episode 71142 total samples 1581471 train steps 49108
Length 114
Episode 71193 total samples 1584017 train steps 49188
Length 111
Episode 71233 total samples 1586531 train steps 49267
Length 106
Episode 71274 total samples 1589039 train steps 49345
Length 24
Episode 71314 total samples 1591560 train steps 49424
Length 92
Episode 71356 total samples 1594089 train steps 49503
Length 66
Episode 71393 total samples 1596594 train steps 49581
Length 389
Episode 71437 total samples 1599114 train steps 49660
Length 148
Episode 71486 total samples 1601628 train steps 49738
Length 104
exploration 0.0999991000096
Episode 71525 total samples 1604158 train steps 49817
Length 285
Episode 71566 total samples 1606713 train steps 49897
Length 114
Episode 71621 total samples 1609224 train steps 49976
Length 115
Episode 71659 total samples 1611745 train steps 50055
Length 127
Episode 71726 total samples 1614316 train steps 50135
Length 242
Episode 71767 total samples 1616889 train steps 50215
Length 159
Episode 71812 total samples 1619414 train steps 50294
Length 190
Episode 71854 total samples 1621938 train steps 50373
Length 96
Episode 71899 total samples 1624547 train steps 50455
Length 264
exploration 0.0999991000096
Episode 71934 total samples 1627176 train steps 50537
Length 431
Episode 71973 total samples 1629728 train steps 50616
Length 175
Episode 72018 total samples 1632297 train steps 50697
Length 140
Episode 72083 total samples 1634850 train steps 50777
Length 147
Episode 72119 total samples 1637411 train steps 50857
Length 444
Episode 72162 total samples 1639948 train steps 50936
Length 139
Episode 72200 total samples 1642476 train steps 51015
Length 249
Episode 72238 total samples 1645018 train steps 51094
Length 188
Episode 72290 total samples 1647582 train steps 51174
Length 161
Episode 72326 total samples 1650128 train steps 51254
Length 326
exploration 0.0999991000096
Episode 72376 total samples 1652640 train steps 51332
Length 104
Episode 72402 total samples 1655166 train steps 51411
Length 115
Episode 72427 total samples 1657709 train steps 51491
Length 353
Episode 72483 total samples 1660266 train steps 51571
Length 136
Episode 72526 total samples 1662883 train steps 51653
Length 282
Episode 72564 total samples 1665390 train steps 51731
Length 35
Episode 72595 total samples 1667994 train steps 51812
Length 391
Episode 72633 total samples 1670519 train steps 51891
Length 189
Episode 72666 total samples 1673025 train steps 51970
Length 90
Episode 72708 total samples 1675532 train steps 52048
Length 56
exploration 0.0999991000096
Episode 72775 total samples 1678034 train steps 52126
Length 44
Episode 72833 total samples 1680576 train steps 52205
Length 115
Episode 72891 total samples 1683128 train steps 52285
Length 133
Episode 72935 total samples 1685633 train steps 52364
Length 164
Episode 72971 total samples 1688266 train steps 52446
Length 375
Episode 73013 total samples 1690795 train steps 52525
Length 154
Episode 73060 total samples 1693303 train steps 52603
Length 55
Episode 73144 total samples 1695849 train steps 52683
Length 145
Episode 73185 total samples 1698359 train steps 52761
Length 105
Episode 73222 total samples 1700928 train steps 52841
Length 141
exploration 0.0999991000096
Episode 73256 total samples 1703457 train steps 52921
Length 165
Episode 73294 total samples 1706003 train steps 53000
Length 183
Episode 73350 total samples 1708541 train steps 53079
Length 96
Episode 73377 total samples 1711082 train steps 53159
Length 293
Episode 73402 total samples 1713642 train steps 53239
Length 165
Episode 73435 total samples 1716193 train steps 53319
Length 187
Episode 73472 total samples 1718819 train steps 53401
Length 420
Episode 73513 total samples 1721428 train steps 53482
Length 382
Episode 73552 total samples 1723981 train steps 53562
Length 322
Episode 73592 total samples 1726579 train steps 53643
Length 255
exploration 0.0999991000096
Episode 73627 total samples 1729084 train steps 53721
Length 289
Episode 73665 total samples 1731592 train steps 53800
Length 97
Episode 73717 total samples 1734099 train steps 53878
Length 413
Episode 73753 total samples 1736717 train steps 53960
Length 265
Episode 73793 total samples 1739330 train steps 54042
Length 294
Episode 73835 total samples 1741840 train steps 54120
Length 120
Episode 73863 total samples 1744419 train steps 54201
Length 227
Episode 73902 total samples 1747015 train steps 54282
Length 262
Episode 73947 total samples 1749643 train steps 54364
Length 458
Episode 73985 total samples 1752188 train steps 54443
Length 139
exploration 0.0999991000096
Episode 74034 total samples 1754729 train steps 54523
Length 117
Episode 74075 total samples 1757321 train steps 54604
Length 323
Episode 74098 total samples 1759846 train steps 54683
Length 143
Episode 74147 total samples 1762465 train steps 54765
Length 329
Episode 74189 total samples 1765117 train steps 54847
Length 387
Episode 74235 total samples 1767662 train steps 54927
Length 111
Episode 74275 total samples 1770282 train steps 55009
Length 516
Episode 74311 total samples 1772967 train steps 55093
Length 389
Episode 74359 total samples 1775505 train steps 55172
Length 176
exploration 0.0999991000096
Episode 74404 total samples 1778037 train steps 55251
Length 95
Episode 74451 total samples 1780626 train steps 55332
Length 321
Episode 74485 total samples 1783182 train steps 55412
Length 152
Episode 74530 total samples 1785712 train steps 55491
Length 143
Episode 74559 total samples 1788213 train steps 55569
Length 447
Episode 74607 total samples 1790731 train steps 55648
Length 156
Episode 74642 total samples 1793328 train steps 55729
Length 206
Episode 74692 total samples 1795852 train steps 55808
Length 157
Episode 74734 total samples 1798368 train steps 55886
Length 105
Episode 74809 total samples 1800918 train steps 55966
Length 105
exploration 0.0999991000096
Episode 74838 total samples 1803444 train steps 56045
Length 194
Episode 74868 total samples 1805985 train steps 56125
Length 189
Episode 74904 total samples 1808679 train steps 56209
Length 549
Episode 75108 total samples 1811185 train steps 56287
Length 117
Episode 75149 total samples 1813726 train steps 56366
Length 171
Episode 75181 total samples 1816270 train steps 56446
Length 160
Episode 75221 total samples 1818808 train steps 56525
Length 98
Episode 75292 total samples 1821319 train steps 56604
Length 232
Episode 75334 total samples 1823883 train steps 56684
Length 507
Episode 75384 total samples 1826396 train steps 56762
Length 96
exploration 0.0999991000096
Episode 75426 total samples 1828953 train steps 56842
Length 118
Episode 75453 total samples 1831492 train steps 56922
Length 180
Episode 75495 total samples 1834072 train steps 57002
Length 250
Episode 75523 total samples 1836632 train steps 57082
Length 661
Episode 75560 total samples 1839137 train steps 57161
Length 31
Episode 75611 total samples 1841682 train steps 57240
Length 135
Episode 75655 total samples 1844343 train steps 57323
Length 385
Episode 75677 total samples 1846935 train steps 57404
Length 216
Episode 75718 total samples 1849481 train steps 57484
Length 96
Episode 75763 total samples 1852033 train steps 57564
Length 141
exploration 0.0999991000096
Episode 75796 total samples 1854588 train steps 57643
Length 112
Episode 75832 total samples 1857193 train steps 57725
Length 213
Episode 75866 total samples 1859736 train steps 57804
Length 169
Episode 75909 total samples 1862277 train steps 57884
Length 100
Episode 75948 total samples 1864806 train steps 57963
Length 312
Episode 75980 total samples 1867324 train steps 58041
Length 76
Episode 76010 total samples 1869841 train steps 58120
Length 69
Episode 76052 total samples 1872439 train steps 58201
Length 240
Episode 76080 total samples 1875064 train steps 58283
Length 486
exploration 0.0999991000096
Episode 76130 total samples 1877572 train steps 58362
Length 84
Episode 76173 total samples 1880106 train steps 58441
Length 112
Episode 76216 total samples 1882628 train steps 58520
Length 213
Episode 76248 total samples 1885218 train steps 58601
Length 258
Episode 76283 total samples 1887818 train steps 58682
Length 219
Episode 76314 total samples 1890338 train steps 58761
Length 68
Episode 76346 total samples 1892880 train steps 58840
Length 191
Episode 76379 total samples 1895389 train steps 58918
Length 61
Episode 76412 total samples 1897894 train steps 58997
Length 122
Episode 76459 total samples 1900420 train steps 59076
Length 52
exploration 0.0999991000096
Episode 76488 total samples 1902971 train steps 59155
Length 177
Episode 76510 total samples 1905484 train steps 59234
Length 96
Episode 76531 total samples 1908230 train steps 59320
Length 819
Episode 76551 total samples 1910800 train steps 59400
Length 152
Episode 76584 total samples 1913447 train steps 59483
Length 357
Episode 76618 total samples 1916101 train steps 59566
Length 366
Episode 76649 total samples 1918668 train steps 59646
Length 435
Episode 76676 total samples 1921184 train steps 59724
Length 96
Episode 76709 total samples 1923871 train steps 59808
Length 548
Episode 76743 total samples 1926416 train steps 59888
Length 120
exploration 0.0999991000096
Episode 76784 total samples 1929030 train steps 59970
Length 340
Episode 76821 total samples 1931564 train steps 60049
Length 101
Episode 76859 total samples 1934237 train steps 60132
Length 411
Episode 76891 total samples 1936785 train steps 60212
Length 300
Episode 76921 total samples 1939337 train steps 60292
Length 227
Episode 76960 total samples 1941935 train steps 60373
Length 234
Episode 76991 total samples 1944471 train steps 60452
Length 127
Episode 77032 total samples 1947158 train steps 60536
Length 400
Episode 77053 total samples 1949682 train steps 60615
Length 238
Episode 77083 total samples 1952262 train steps 60696
Length 180
exploration 0.0999991000096
Episode 77126 total samples 1955243 train steps 60789
Length 1062
Episode 77148 total samples 1957772 train steps 60868
Length 169
Episode 77173 total samples 1960421 train steps 60951
Length 319
Episode 77202 total samples 1963001 train steps 61031
Length 230
Episode 77222 total samples 1965545 train steps 61111
Length 235
Episode 77252 total samples 1968114 train steps 61191
Length 222
Episode 77276 total samples 1970637 train steps 61270
Length 134
Episode 77300 total samples 1973149 train steps 61348
Length 99
Episode 77346 total samples 1975710 train steps 61428
Length 199
exploration 0.0999991000096
Episode 77370 total samples 1978371 train steps 61512
Length 398
Episode 77407 total samples 1980945 train steps 61592
Length 169
Episode 77436 total samples 1983464 train steps 61671
Length 200
Episode 77458 total samples 1986059 train steps 61752
Length 473
Episode 77480 total samples 1988568 train steps 61830
Length 165
Episode 77511 total samples 1991233 train steps 61914
Length 346
Episode 77536 total samples 1993761 train steps 61993
Length 85
Episode 77569 total samples 1996399 train steps 62075
Length 595
Episode 77594 total samples 1998930 train steps 62154
Length 195
Episode 77621 total samples 2001624 train steps 62238
Length 391
exploration 0.0999991000096
Episode 77651 total samples 2004146 train steps 62317
Length 67
Episode 77684 total samples 2006652 train steps 62395
Length 82
Episode 77707 total samples 2009188 train steps 62475
Length 225
Episode 77732 total samples 2011760 train steps 62555
Length 281
Episode 77760 total samples 2014305 train steps 62635
Length 102
Episode 77790 total samples 2016829 train steps 62713
Length 155
Episode 77812 total samples 2019439 train steps 62795
Length 272
Episode 77834 total samples 2022243 train steps 62883
Length 704
Episode 77878 total samples 2024745 train steps 62961
Length 62
Episode 77935 total samples 2027250 train steps 63039
Length 79
exploration 0.0999991000096
Episode 77964 total samples 2029782 train steps 63118
Length 110
Episode 77987 total samples 2032340 train steps 63198
Length 151
Episode 78009 total samples 2035056 train steps 63283
Length 702
Episode 78030 total samples 2037565 train steps 63361
Length 51
Episode 78055 total samples 2040221 train steps 63444
Length 1156
Episode 78078 total samples 2042875 train steps 63527
Length 325
Episode 78103 total samples 2045412 train steps 63607
Length 124
Episode 78126 total samples 2048046 train steps 63689
Length 485
Episode 78146 total samples 2050632 train steps 63770
Length 314
exploration 0.0999991000096
Episode 78183 total samples 2053155 train steps 63849
Length 119
Episode 78215 total samples 2055678 train steps 63927
Length 136
Episode 78236 total samples 2058184 train steps 64006
Length 396
Episode 78263 total samples 2060752 train steps 64086
Length 212
Episode 78296 total samples 2063258 train steps 64164
Length 292
Episode 78319 total samples 2065765 train steps 64243
Length 949
Episode 78352 total samples 2068357 train steps 64324
Length 358
Episode 78381 total samples 2070877 train steps 64402
Length 79
Episode 78409 total samples 2073420 train steps 64482
Length 203
Episode 78435 total samples 2075943 train steps 64561
Length 430
exploration 0.0999991000096
Episode 78469 total samples 2078451 train steps 64639
Length 245
Episode 78493 total samples 2080957 train steps 64717
Length 238
Episode 78514 total samples 2083490 train steps 64797
Length 479
Episode 78542 total samples 2086334 train steps 64885
Length 749
Episode 78575 total samples 2088879 train steps 64965
Length 247
Episode 78598 total samples 2091406 train steps 65044
Length 517
Episode 78627 total samples 2094053 train steps 65127
Length 329
Episode 78655 total samples 2096649 train steps 65208
Length 617
Episode 78687 total samples 2099163 train steps 65286
Length 169
Episode 78709 total samples 2101710 train steps 65366
Length 283
exploration 0.0999991000096
Episode 78732 total samples 2104345 train steps 65448
Length 655
Episode 78760 total samples 2106910 train steps 65528
Length 716
Episode 78785 total samples 2109700 train steps 65616
Length 1025
Episode 78814 total samples 2112391 train steps 65700
Length 940
Episode 78842 total samples 2115037 train steps 65782
Length 314
Episode 78876 total samples 2117547 train steps 65861
Length 75
Episode 78904 total samples 2120172 train steps 65943
Length 390
Episode 78929 total samples 2122822 train steps 66026
Length 444
Episode 78950 total samples 2125333 train steps 66104
Length 92
Episode 78985 total samples 2127853 train steps 66183
Length 255
exploration 0.0999991000096
Episode 79016 total samples 2130392 train steps 66262
Length 170
Episode 79041 total samples 2132925 train steps 66341
Length 389
Episode 79060 total samples 2135432 train steps 66420
Length 362
Episode 79098 total samples 2138188 train steps 66506
Length 526
Episode 79120 total samples 2140709 train steps 66585
Length 67
Episode 79147 total samples 2143282 train steps 66665
Length 192
Episode 79167 total samples 2145874 train steps 66746
Length 441
Episode 79187 total samples 2148403 train steps 66825
Length 345
Episode 79214 total samples 2151002 train steps 66906
Length 288
exploration 0.0999991000096
Episode 79237 total samples 2153572 train steps 66987
Length 333
Episode 79254 total samples 2156124 train steps 67066
Length 295
Episode 79275 total samples 2158637 train steps 67145
Length 584
Episode 79295 total samples 2161210 train steps 67225
Length 266
Episode 79320 total samples 2163727 train steps 67304
Length 125
Episode 79356 total samples 2166635 train steps 67395
Length 816
Episode 79375 total samples 2169213 train steps 67475
Length 156
Episode 79407 total samples 2172132 train steps 67567
Length 855
Episode 79435 total samples 2174710 train steps 67647
Length 174
Episode 79466 total samples 2177274 train steps 67727
Length 231
exploration 0.0999991000096
Episode 79492 total samples 2179800 train steps 67806
Length 94
Episode 79522 total samples 2182365 train steps 67886
Length 185
Episode 79540 total samples 2184879 train steps 67965
Length 1091
Episode 79564 total samples 2187442 train steps 68045
Length 526
Episode 79588 total samples 2190012 train steps 68125
Length 152
Episode 79612 total samples 2192702 train steps 68209
Length 401
Episode 79628 total samples 2195256 train steps 68289
Length 245
Episode 79654 total samples 2197862 train steps 68371
Length 216
Episode 79745 total samples 2200395 train steps 68450
Length 84
Episode 79764 total samples 2203112 train steps 68535
Length 541
exploration 0.0999991000096
Episode 79787 total samples 2205744 train steps 68617
Length 555
Episode 79803 total samples 2208331 train steps 68698
Length 193
Episode 79826 total samples 2210834 train steps 68776
Length 299
Episode 79845 total samples 2213362 train steps 68855
Length 203
Episode 79865 total samples 2215866 train steps 68933
Length 197
Episode 79891 total samples 2218369 train steps 69012
Length 35
Episode 79915 total samples 2221145 train steps 69098
Length 903
Episode 79927 total samples 2223662 train steps 69177
Length 257
Episode 79962 total samples 2226210 train steps 69257
Length 184
exploration 0.0999991000096
Episode 79989 total samples 2228775 train steps 69337
Length 567
Episode 80026 total samples 2231284 train steps 69415
Length 140
Episode 80052 total samples 2233812 train steps 69494
Length 348
Episode 80077 total samples 2236359 train steps 69574
Length 668
Episode 80101 total samples 2239010 train steps 69657
Length 642
Episode 80123 total samples 2242649 train steps 69770
Length 2473
Episode 80147 total samples 2245202 train steps 69850
Length 124
Episode 80162 total samples 2247808 train steps 69931
Length 248
Episode 80186 total samples 2250386 train steps 70012
Length 289
Episode 80222 total samples 2253098 train steps 70097
Length 526
exploration 0.0999991000096
Episode 80246 total samples 2255675 train steps 70177
Length 639
Episode 80268 total samples 2258354 train steps 70261
Length 365
Episode 80295 total samples 2260969 train steps 70343
Length 326
Episode 80333 total samples 2263555 train steps 70424
Length 197
Episode 80367 total samples 2266190 train steps 70506
Length 324
Episode 80386 total samples 2268691 train steps 70584
Length 40
Episode 80411 total samples 2271324 train steps 70666
Length 361
Episode 80435 total samples 2273864 train steps 70746
Length 130
Episode 80454 total samples 2276399 train steps 70825
Length 141
exploration 0.0999991000096
Episode 80486 total samples 2278913 train steps 70904
Length 35
Episode 80505 total samples 2281444 train steps 70983
Length 270
Episode 80546 total samples 2284020 train steps 71063
Length 215
Episode 80571 total samples 2286577 train steps 71143
Length 485
Episode 80590 total samples 2289109 train steps 71222
Length 92
Episode 80608 total samples 2291680 train steps 71302
Length 586
Episode 80636 total samples 2294214 train steps 71382
Length 697
Episode 80663 total samples 2296789 train steps 71462
Length 255
Episode 80720 total samples 2299336 train steps 71542
Length 260
Episode 80747 total samples 2301910 train steps 71622
Length 317
exploration 0.0999991000096
Episode 80785 total samples 2304771 train steps 71712
Length 739
Episode 80804 total samples 2307291 train steps 71790
Length 119
Episode 80828 total samples 2309941 train steps 71873
Length 508
Episode 80846 total samples 2312565 train steps 71955
Length 275
Episode 80873 total samples 2315117 train steps 72035
Length 189
Episode 80893 total samples 2317637 train steps 72114
Length 615
Episode 80914 total samples 2320333 train steps 72198
Length 831
Episode 80935 total samples 2323247 train steps 72289
Length 965
Episode 80953 total samples 2325950 train steps 72373
Length 502
Episode 80967 total samples 2328490 train steps 72453
Length 277
exploration 0.0999991000096
Episode 80994 total samples 2331262 train steps 72539
Length 554
Episode 81028 total samples 2333780 train steps 72618
Length 100
Episode 81051 total samples 2336293 train steps 72697
Length 179
Episode 81075 total samples 2338844 train steps 72776
Length 446
Episode 81102 total samples 2341673 train steps 72865
Length 783
Episode 81129 total samples 2344246 train steps 72945
Length 409
Episode 81157 total samples 2346920 train steps 73029
Length 417
Episode 81182 total samples 2349581 train steps 73112
Length 334
Episode 81217 total samples 2352159 train steps 73192
Length 259
exploration 0.0999991000096
Episode 81245 total samples 2354737 train steps 73273
Length 397
Episode 81272 total samples 2357259 train steps 73352
Length 173
Episode 81291 total samples 2359813 train steps 73432
Length 309
Episode 81312 total samples 2362369 train steps 73512
Length 198
Episode 81340 total samples 2365027 train steps 73595
Length 394
Episode 81366 total samples 2367549 train steps 73673
Length 175
Episode 81391 total samples 2370102 train steps 73753
Length 563
Episode 81416 total samples 2372774 train steps 73837
Length 350
Episode 81441 total samples 2375379 train steps 73918
Length 276
Episode 81469 total samples 2378000 train steps 74000
Length 267
exploration 0.0999991000096
Episode 81489 total samples 2380764 train steps 74086
Length 870
Episode 81514 total samples 2383344 train steps 74167
Length 465
Episode 81537 total samples 2386170 train steps 74255
Length 760
Episode 81570 total samples 2388683 train steps 74334
Length 34
Episode 81596 total samples 2391262 train steps 74414
Length 280
Episode 81621 total samples 2393769 train steps 74493
Length 520
Episode 81649 total samples 2396308 train steps 74572
Length 445
Episode 81677 total samples 2398917 train steps 74654
Length 641
Episode 81704 total samples 2401615 train steps 74738
Length 480
exploration 0.0999991000096
Episode 81722 total samples 2404163 train steps 74818
Length 124
Episode 81746 total samples 2406902 train steps 74903
Length 494
Episode 81764 total samples 2409603 train steps 74988
Length 487
Episode 81782 total samples 2412118 train steps 75066
Length 188
Episode 81807 total samples 2414774 train steps 75149
Length 633
Episode 81831 total samples 2417404 train steps 75231
Length 292
Episode 81865 total samples 2419943 train steps 75311
Length 255
Episode 81890 total samples 2422489 train steps 75390
Length 672
Episode 81915 total samples 2425032 train steps 75470
Length 359
Episode 81932 total samples 2427533 train steps 75548
Length 111
exploration 0.0999991000096
Episode 81955 total samples 2430093 train steps 75628
Length 149
Episode 81975 total samples 2432822 train steps 75713
Length 933
Episode 81992 total samples 2435402 train steps 75794
Length 286
Episode 82016 total samples 2438078 train steps 75877
Length 602
Episode 82048 total samples 2440619 train steps 75957
Length 226
Episode 82069 total samples 2443157 train steps 76036
Length 273
Episode 82089 total samples 2445690 train steps 76115
Length 67
Episode 82106 total samples 2448199 train steps 76194
Length 408
Episode 82135 total samples 2450719 train steps 76272
Length 42
Episode 82165 total samples 2453559 train steps 76361
Length 999
exploration 0.0999991000096
Episode 82183 total samples 2456064 train steps 76439
Length 65
Episode 82208 total samples 2458646 train steps 76520
Length 305
Episode 82231 total samples 2461307 train steps 76603
Length 681
Episode 82255 total samples 2463918 train steps 76685
Length 392
Episode 82288 total samples 2466499 train steps 76766
Length 184
Episode 82309 total samples 2469034 train steps 76845
Length 114
Episode 82338 total samples 2471542 train steps 76923
Length 51
Episode 82355 total samples 2474067 train steps 77002
Length 82
Episode 82382 total samples 2476575 train steps 77080
Length 143
Episode 82404 total samples 2479279 train steps 77165
Length 606
exploration 0.0999991000096
Episode 82434 total samples 2481836 train steps 77245
Length 526
Episode 82458 total samples 2484400 train steps 77325
Length 433
Episode 82486 total samples 2486958 train steps 77405
Length 843
Episode 82511 total samples 2489538 train steps 77486
Length 339
Episode 82533 total samples 2492145 train steps 77567
Length 421
Episode 82554 total samples 2494670 train steps 77646
Length 126
Episode 82572 total samples 2497345 train steps 77730
Length 516
Episode 82597 total samples 2499943 train steps 77811
Length 416
Episode 82634 total samples 2502535 train steps 77892
Length 513
exploration 0.0999991000096
Episode 82662 total samples 2505117 train steps 77972
Length 223
Episode 82691 total samples 2507774 train steps 78055
Length 349
Episode 82719 total samples 2510298 train steps 78134
Length 107
Episode 82760 total samples 2512852 train steps 78214
Length 269
Episode 82778 total samples 2515370 train steps 78293
Length 173
Episode 82799 total samples 2517875 train steps 78371
Length 150
Episode 82813 total samples 2520379 train steps 78449
Length 254
Episode 82835 total samples 2522945 train steps 78530
Length 357
Episode 82852 total samples 2525557 train steps 78611
Length 761
Episode 82868 total samples 2528095 train steps 78690
Length 343
exploration 0.0999991000096
Episode 82893 total samples 2530617 train steps 78769
Length 68
Episode 82922 total samples 2533120 train steps 78847
Length 18
Episode 82961 total samples 2535749 train steps 78930
Length 466
Episode 82985 total samples 2538309 train steps 79010
Length 173
Episode 83004 total samples 2540993 train steps 79094
Length 377
Episode 83034 total samples 2543632 train steps 79176
Length 339
Episode 83056 total samples 2546254 train steps 79258
Length 253
Episode 83088 total samples 2548833 train steps 79339
Length 211
Episode 83109 total samples 2551592 train steps 79425
Length 1491
Episode 83128 total samples 2554112 train steps 79503
Length 113
exploration 0.0999991000096
Episode 83148 total samples 2556847 train steps 79589
Length 624
Episode 83173 total samples 2559360 train steps 79667
Length 171
Episode 83195 total samples 2561972 train steps 79749
Length 270
Episode 83219 total samples 2564682 train steps 79834
Length 731
Episode 83258 total samples 2567193 train steps 79912
Length 48
Episode 83342 total samples 2569759 train steps 79992
Length 181
Episode 83362 total samples 2572660 train steps 80083
Length 882
Episode 83372 total samples 2575194 train steps 80162
Length 302
Episode 83392 total samples 2577709 train steps 80241
Length 115
exploration 0.0999991000096
Episode 83416 total samples 2580322 train steps 80323
Length 256
Episode 83436 total samples 2582893 train steps 80403
Length 681
Episode 83482 total samples 2585625 train steps 80488
Length 485
Episode 83502 total samples 2588241 train steps 80570
Length 418
Episode 83534 total samples 2590967 train steps 80655
Length 744
Episode 83553 total samples 2593520 train steps 80735
Length 110
Episode 83580 total samples 2596104 train steps 80816
Length 210
Episode 83601 total samples 2598691 train steps 80897
Length 454
Episode 83623 total samples 2601463 train steps 80983
Length 709
Episode 83646 total samples 2604105 train steps 81066
Length 492
exploration 0.0999991000096
Episode 83668 total samples 2606753 train steps 81149
Length 432
Episode 83691 total samples 2609286 train steps 81228
Length 95
Episode 83708 total samples 2611837 train steps 81307
Length 606
Episode 83742 total samples 2614437 train steps 81389
Length 451
Episode 83764 total samples 2617096 train steps 81472
Length 1522
Episode 83795 total samples 2619731 train steps 81554
Length 475
Episode 83814 total samples 2622275 train steps 81634
Length 173
Episode 83833 total samples 2624848 train steps 81714
Length 174
Episode 83856 total samples 2627379 train steps 81793
Length 287
exploration 0.0999991000096
Episode 83888 total samples 2629889 train steps 81872
Length 49
Episode 83913 total samples 2632464 train steps 81952
Length 365
Episode 83936 total samples 2634983 train steps 82031
Length 1074
Episode 83961 total samples 2637510 train steps 82110
Length 263
Episode 83983 total samples 2640043 train steps 82189
Length 175
Episode 84007 total samples 2642610 train steps 82269
Length 657
Episode 84027 total samples 2645148 train steps 82348
Length 128
Episode 84099 total samples 2647849 train steps 82433
Length 485
Episode 84129 total samples 2650438 train steps 82514
Length 301
Episode 84154 total samples 2652950 train steps 82592
Length 82
exploration 0.0999991000096
Episode 84184 total samples 2655523 train steps 82673
Length 213
Episode 84205 total samples 2658231 train steps 82757
Length 1090
Episode 84228 total samples 2660799 train steps 82837
Length 395
Episode 84267 total samples 2663507 train steps 82922
Length 617
Episode 84284 total samples 2666242 train steps 83008
Length 573
Episode 84308 total samples 2668986 train steps 83093
Length 734
Episode 84331 total samples 2671949 train steps 83186
Length 1043
Episode 84351 total samples 2674490 train steps 83265
Length 100
Episode 84374 total samples 2677062 train steps 83346
Length 392
Episode 84397 total samples 2679588 train steps 83425
Length 250
exploration 0.0999991000096
Episode 84420 total samples 2682217 train steps 83507
Length 357
Episode 84438 total samples 2684719 train steps 83585
Length 333
Episode 84467 total samples 2687228 train steps 83663
Length 83
Episode 84484 total samples 2689885 train steps 83746
Length 702
Episode 84504 total samples 2692780 train steps 83837
Length 867
Episode 84534 total samples 2695360 train steps 83917
Length 173
Episode 84566 total samples 2697892 train steps 83997
Length 139
Episode 84585 total samples 2700446 train steps 84076
Length 342
Episode 84601 total samples 2703122 train steps 84160
Length 1048
exploration 0.0999991000096
Episode 84614 total samples 2705684 train steps 84240
Length 227
Episode 84629 total samples 2708221 train steps 84319
Length 411
Episode 84652 total samples 2710895 train steps 84403
Length 418
Episode 84676 total samples 2713432 train steps 84482
Length 84
Episode 84708 total samples 2716275 train steps 84571
Length 694
Episode 84732 total samples 2718787 train steps 84650
Length 23
Episode 84761 total samples 2721343 train steps 84729
Length 201
Episode 84778 total samples 2723999 train steps 84812
Length 662
Episode 84806 total samples 2726554 train steps 84892
Length 319
Episode 84838 total samples 2729109 train steps 84972
Length 243
exploration 0.0999991000096
Episode 84861 total samples 2731676 train steps 85052
Length 350
Episode 84880 total samples 2734200 train steps 85131
Length 435
Episode 84898 total samples 2736908 train steps 85216
Length 481
Episode 84919 total samples 2739446 train steps 85295
Length 367
Episode 84935 total samples 2742097 train steps 85378
Length 699
Episode 84950 total samples 2744709 train steps 85460
Length 772
Episode 84969 total samples 2747304 train steps 85541
Length 314
Episode 85003 total samples 2749910 train steps 85622
Length 481
Episode 85020 total samples 2752429 train steps 85701
Length 128
Episode 85031 total samples 2755007 train steps 85781
Length 410
exploration 0.0999991000096
Episode 85047 total samples 2757551 train steps 85861
Length 209
Episode 85065 total samples 2760132 train steps 85942
Length 381
Episode 85084 total samples 2763038 train steps 86032
Length 1029
Episode 85107 total samples 2765540 train steps 86111
Length 58
Episode 85122 total samples 2768212 train steps 86194
Length 506
Episode 85141 total samples 2770793 train steps 86275
Length 171
Episode 85154 total samples 2773499 train steps 86359
Length 539
Episode 85168 total samples 2776031 train steps 86438
Length 512
Episode 85188 total samples 2778633 train steps 86520
Length 224
exploration 0.0999991000096
Episode 85205 total samples 2781353 train steps 86605
Length 908
Episode 85226 total samples 2783913 train steps 86685
Length 297
Episode 85255 total samples 2786471 train steps 86765
Length 245
Episode 85276 total samples 2789098 train steps 86847
Length 286
Episode 85302 total samples 2791641 train steps 86926
Length 419
Episode 85323 total samples 2794166 train steps 87005
Length 618
Episode 85343 total samples 2796887 train steps 87090
Length 1056
Episode 85365 total samples 2799425 train steps 87170
Length 86
Episode 85401 total samples 2801964 train steps 87249
Length 99
Episode 85418 total samples 2804606 train steps 87331
Length 507
exploration 0.0999991000096
Episode 85444 total samples 2807254 train steps 87414
Length 383
Episode 85455 total samples 2809771 train steps 87493
Length 518
Episode 85480 total samples 2812282 train steps 87571
Length 32
Episode 85503 total samples 2814824 train steps 87651
Length 560
Episode 85528 total samples 2817481 train steps 87734
Length 666
Episode 85553 total samples 2820009 train steps 87813
Length 254
Episode 85575 total samples 2822626 train steps 87895
Length 302
Episode 85603 total samples 2825350 train steps 87980
Length 497
Episode 85619 total samples 2827963 train steps 88061
Length 562
Episode 85648 total samples 2831170 train steps 88162
Length 1614
exploration 0.0999991000096
Episode 85670 total samples 2833764 train steps 88243
Length 245
Episode 85688 total samples 2836302 train steps 88322
Length 115
Episode 85729 total samples 2838873 train steps 88402
Length 198
Episode 85744 total samples 2841433 train steps 88482
Length 166
Episode 85774 total samples 2844068 train steps 88565
Length 284
Episode 85795 total samples 2846606 train steps 88644
Length 889
Episode 85815 total samples 2849231 train steps 88726
Length 306
Episode 85835 total samples 2851804 train steps 88806
Length 380
Episode 85848 total samples 2854455 train steps 88889
Length 1263
exploration 0.0999991000096
Episode 85873 total samples 2857000 train steps 88969
Length 274
Episode 85892 total samples 2859587 train steps 89050
Length 607
Episode 85911 total samples 2862186 train steps 89131
Length 226
Episode 85938 total samples 2864940 train steps 89217
Length 804
Episode 85951 total samples 2867495 train steps 89297
Length 479
Episode 85967 total samples 2870034 train steps 89376
Length 322
Episode 85996 total samples 2872776 train steps 89462
Length 1154
Episode 86019 total samples 2875288 train steps 89540
Length 319
Episode 86041 total samples 2877965 train steps 89624
Length 549
Episode 86060 total samples 2880651 train steps 89708
Length 579
exploration 0.0999991000096
Episode 86075 total samples 2883197 train steps 89787
Length 147
Episode 86091 total samples 2885888 train steps 89871
Length 617
Episode 86111 total samples 2888438 train steps 89951
Length 430
Episode 86129 total samples 2891529 train steps 90048
Length 1591
Episode 86151 total samples 2894307 train steps 90135
Length 597
Episode 86170 total samples 2897041 train steps 90220
Length 679
Episode 86200 total samples 2899570 train steps 90299
Length 183
Episode 86226 total samples 2902126 train steps 90379
Length 135
Episode 86263 total samples 2904740 train steps 90461
Length 323
exploration 0.0999991000096
Episode 86289 total samples 2907445 train steps 90545
Length 492
Episode 86312 total samples 2909951 train steps 90623
Length 580
Episode 86336 total samples 2912569 train steps 90705
Length 452
Episode 86358 total samples 2915277 train steps 90790
Length 773
Episode 86375 total samples 2917966 train steps 90874
Length 653
Episode 86394 total samples 2920624 train steps 90957
Length 859
Episode 86415 total samples 2923146 train steps 91036
Length 56
Episode 86447 total samples 2925752 train steps 91117
Length 861
Episode 86477 total samples 2928289 train steps 91197
Length 150
Episode 86494 total samples 2930863 train steps 91277
Length 583
exploration 0.0999991000096
Episode 86533 total samples 2933425 train steps 91357
Length 268
Episode 86597 total samples 2935949 train steps 91436
Length 141
Episode 86632 total samples 2938678 train steps 91521
Length 512
Episode 86653 total samples 2941335 train steps 91604
Length 639
Episode 86679 total samples 2943983 train steps 91687
Length 596
Episode 86694 total samples 2946583 train steps 91768
Length 564
Episode 86986 total samples 2949084 train steps 91846
Length 9
Episode 87287 total samples 2951588 train steps 91925
Length 10
Episode 87377 total samples 2954115 train steps 92004
Length 160
Episode 87436 total samples 2956685 train steps 92084
Length 164
exploration 0.0999991000096
Episode 87469 total samples 2959262 train steps 92164
Length 485
Episode 87488 total samples 2961956 train steps 92249
Length 564
Episode 87508 total samples 2964513 train steps 92329
Length 239
Episode 87529 total samples 2967105 train steps 92410
Length 297
Episode 87555 total samples 2969676 train steps 92490
Length 159
Episode 87578 total samples 2972204 train steps 92569
Length 412
Episode 87596 total samples 2974909 train steps 92653
Length 453
Episode 87622 total samples 2977537 train steps 92736
Length 303
Episode 87652 total samples 2980230 train steps 92820
Length 773
exploration 0.0999991000096
Episode 87674 total samples 2982829 train steps 92901
Length 203
Episode 87714 total samples 2985429 train steps 92982
Length 206
Episode 87744 total samples 2987991 train steps 93062
Length 344
Episode 87774 total samples 2990520 train steps 93141
Length 149
Episode 87789 total samples 2993025 train steps 93220
Length 1648
Episode 87811 total samples 2995625 train steps 93301
Length 502
Episode 87866 total samples 2998301 train steps 93384
Length 540
Episode 87880 total samples 3000853 train steps 93464
Length 829
Episode 87901 total samples 3003360 train steps 93542
Length 81
Episode 87919 total samples 3006015 train steps 93625
Length 386
exploration 0.0999991000096
Episode 87942 total samples 3008549 train steps 93705
Length 91
Episode 87967 total samples 3011067 train steps 93783
Length 77
Episode 88048 total samples 3013607 train steps 93863
Length 86
Episode 88076 total samples 3016139 train steps 93942
Length 748
Episode 88089 total samples 3018702 train steps 94022
Length 283
Episode 88115 total samples 3021315 train steps 94104
Length 498
Episode 88142 total samples 3023947 train steps 94186
Length 394
Episode 88170 total samples 3026452 train steps 94264
Length 291
Episode 88182 total samples 3028987 train steps 94343
Length 442
Episode 88209 total samples 3031610 train steps 94425
Length 245
exploration 0.0999991000096
Episode 88238 total samples 3034149 train steps 94505
Length 79
Episode 88262 total samples 3036674 train steps 94584
Length 209
Episode 88278 total samples 3039552 train steps 94673
Length 817
Episode 88295 total samples 3042193 train steps 94756
Length 457
Episode 88323 total samples 3044718 train steps 94835
Length 188
Episode 88340 total samples 3047280 train steps 94915
Length 529
Episode 88364 total samples 3049827 train steps 94995
Length 461
Episode 88397 total samples 3052333 train steps 95073
Length 82
Episode 88428 total samples 3055116 train steps 95160
Length 745
exploration 0.0999991000096
Episode 88449 total samples 3057637 train steps 95239
Length 280
Episode 88470 total samples 3060225 train steps 95320
Length 345
Episode 88494 total samples 3063033 train steps 95407
Length 789
Episode 88522 total samples 3065623 train steps 95488
Length 347
Episode 88548 total samples 3068547 train steps 95580
Length 1353
Episode 88562 total samples 3071113 train steps 95660
Length 442
Episode 88588 total samples 3073966 train steps 95749
Length 1133
Episode 88609 total samples 3076495 train steps 95828
Length 243
Episode 88626 total samples 3079089 train steps 95909
Length 982
Episode 88643 total samples 3081660 train steps 95989
Length 153
exploration 0.0999991000096
Episode 88667 total samples 3084670 train steps 96083
Length 1046
Episode 88686 total samples 3087312 train steps 96166
Length 601
Episode 88715 total samples 3089817 train steps 96244
Length 126
Episode 88737 total samples 3092383 train steps 96324
Length 199
Episode 88764 total samples 3094888 train steps 96403
Length 339
Episode 88785 total samples 3097497 train steps 96484
Length 280
Episode 88809 total samples 3100077 train steps 96565
Length 349
Episode 88826 total samples 3102658 train steps 96646
Length 611
Episode 88849 total samples 3105176 train steps 96724
Length 527
exploration 0.0999991000096
Episode 88880 total samples 3107736 train steps 96804
Length 404
Episode 88906 total samples 3110355 train steps 96886
Length 365
Episode 88926 total samples 3112933 train steps 96967
Length 418
Episode 88944 total samples 3115640 train steps 97051
Length 980
Episode 88964 total samples 3118151 train steps 97130
Length 145
Episode 88988 total samples 3120905 train steps 97216
Length 542
Episode 89008 total samples 3123527 train steps 97298
Length 274
Episode 89036 total samples 3126041 train steps 97376
Length 194
Episode 89054 total samples 3128572 train steps 97455
Length 364
Episode 89073 total samples 3131299 train steps 97541
Length 545
exploration 0.0999991000096
Episode 89122 total samples 3133823 train steps 97619
Length 165
Episode 89144 total samples 3136352 train steps 97698
Length 110
Episode 89174 total samples 3138977 train steps 97781
Length 754
Episode 89198 total samples 3141531 train steps 97860
Length 232
Episode 89225 total samples 3144111 train steps 97941
Length 394
Episode 89240 total samples 3146829 train steps 98026
Length 975
Episode 89264 total samples 3149481 train steps 98109
Length 309
Episode 89289 total samples 3152034 train steps 98189
Length 273
Episode 89315 total samples 3154658 train steps 98271
Length 254
Episode 89334 total samples 3157191 train steps 98350
Length 108
exploration 0.0999991000096
Episode 89358 total samples 3159710 train steps 98428
Length 383
Episode 89386 total samples 3162286 train steps 98509
Length 494
Episode 89406 total samples 3164853 train steps 98589
Length 308
Episode 89429 total samples 3167418 train steps 98669
Length 155
Episode 89450 total samples 3169930 train steps 98748
Length 110
Episode 89469 total samples 3172587 train steps 98831
Length 471
Episode 89494 total samples 3175090 train steps 98909
Length 25
Episode 89523 total samples 3177689 train steps 98990
Length 250
Episode 89546 total samples 3180212 train steps 99069
Length 737
exploration 0.0999991000096
Episode 89571 total samples 3182737 train steps 99148
Length 118
Episode 89603 total samples 3185283 train steps 99228
Length 157
Episode 89626 total samples 3187866 train steps 99308
Length 381
Episode 89649 total samples 3190416 train steps 99388
Length 149
Episode 89669 total samples 3192947 train steps 99467
Length 188
Episode 89695 total samples 3195580 train steps 99549
Length 443
Episode 89717 total samples 3198117 train steps 99629
Length 81
Episode 89740 total samples 3200930 train steps 99717
Length 632
Episode 89763 total samples 3203446 train steps 99795
Length 61
Episode 89783 total samples 3206038 train steps 99876
Length 304
exploration 0.0999991000096
Episode 89805 total samples 3208713 train steps 99960
Length 353
Episode 89836 total samples 3211280 train steps 100040
Length 209
Episode 89866 total samples 3213897 train steps 100122
Length 291
Episode 89885 total samples 3216400 train steps 100200
Length 28
Episode 89925 total samples 3218969 train steps 100280
Length 221
Episode 89947 total samples 3221701 train steps 100366
Length 681
Episode 89962 total samples 3224291 train steps 100447
Length 460
Episode 89992 total samples 3226927 train steps 100529
Length 334
Episode 90004 total samples 3229527 train steps 100610
Length 228
Episode 90031 total samples 3232042 train steps 100689
Length 70
exploration 0.0999991000096
Episode 90065 total samples 3235140 train steps 100786
Length 1239
Episode 90085 total samples 3237763 train steps 100868
Length 290
Episode 90100 total samples 3240481 train steps 100953
Length 487
Episode 90121 total samples 3243439 train steps 101045
Length 1501
Episode 90137 total samples 3245946 train steps 101123
Length 194
Episode 90158 total samples 3248654 train steps 101208
Length 482
Episode 90185 total samples 3251202 train steps 101288
Length 248
Episode 90203 total samples 3254081 train steps 101378
Length 858
Episode 90227 total samples 3256650 train steps 101458
Length 421
exploration 0.0999991000096
Episode 90259 total samples 3259247 train steps 101539
Length 672
Episode 90287 total samples 3261889 train steps 101622
Length 318
Episode 90309 total samples 3264578 train steps 101706
Length 661
Episode 90351 total samples 3267125 train steps 101785
Length 95
Episode 90407 total samples 3269665 train steps 101865
Length 124
Episode 90441 total samples 3272169 train steps 101943
Length 846
Episode 90471 total samples 3274860 train steps 102027
Length 446
Episode 90495 total samples 3277393 train steps 102106
Length 107
Episode 90510 total samples 3279930 train steps 102185
Length 524
Episode 90531 total samples 3282538 train steps 102267
Length 287
exploration 0.0999991000096
Episode 90567 total samples 3285154 train steps 102349
Length 357
Episode 90596 total samples 3287678 train steps 102427
Length 499
Episode 90628 total samples 3290189 train steps 102506
Length 132
Episode 90709 total samples 3292746 train steps 102586
Length 208
Episode 90739 total samples 3295259 train steps 102664
Length 289
Episode 90755 total samples 3297878 train steps 102746
Length 322
Episode 90774 total samples 3300479 train steps 102827
Length 219
Episode 90800 total samples 3303063 train steps 102908
Length 363
Episode 90830 total samples 3305692 train steps 102990
Length 799
exploration 0.0999991000096
Episode 90852 total samples 3308222 train steps 103069
Length 414
Episode 90881 total samples 3310729 train steps 103148
Length 63
Episode 90902 total samples 3313236 train steps 103226
Length 92
Episode 90935 total samples 3315778 train steps 103306
Length 83
Episode 90960 total samples 3318364 train steps 103386
Length 302
Episode 90979 total samples 3320913 train steps 103466
Length 311
Episode 91002 total samples 3323508 train steps 103547
Length 256
Episode 91031 total samples 3326056 train steps 103627
Length 476
Episode 91142 total samples 3328562 train steps 103705
Length 11
Episode 91223 total samples 3331186 train steps 103787
Length 310
exploration 0.0999991000096
Episode 91246 total samples 3333722 train steps 103866
Length 199
Episode 91291 total samples 3336226 train steps 103945
Length 44
Episode 91321 total samples 3338846 train steps 104026
Length 537
Episode 91346 total samples 3341401 train steps 104106
Length 197
Episode 91383 total samples 3344007 train steps 104188
Length 212
Episode 91411 total samples 3346577 train steps 104268
Length 285
Episode 91439 total samples 3349098 train steps 104347
Length 70
Episode 91466 total samples 3351612 train steps 104425
Length 359
Episode 91481 total samples 3354194 train steps 104506
Length 874
Episode 91500 total samples 3356987 train steps 104593
Length 884
exploration 0.0999991000096
Episode 91514 total samples 3359551 train steps 104673
Length 679
Episode 91541 total samples 3362266 train steps 104758
Length 599
Episode 91562 total samples 3364982 train steps 104843
Length 731
Episode 91575 total samples 3367706 train steps 104928
Length 1090
Episode 91589 total samples 3370238 train steps 105007
Length 601
Episode 91602 total samples 3372784 train steps 105087
Length 687
Episode 91626 total samples 3375444 train steps 105170
Length 395
Episode 91646 total samples 3378213 train steps 105257
Length 860
Episode 91677 total samples 3380946 train steps 105342
Length 531
exploration 0.0999991000096
Episode 91701 total samples 3383463 train steps 105421
Length 178
Episode 91720 total samples 3386040 train steps 105501
Length 341
Episode 91739 total samples 3388566 train steps 105580
Length 207
Episode 91764 total samples 3391129 train steps 105660
Length 160
Episode 91787 total samples 3393733 train steps 105742
Length 257
Episode 91811 total samples 3396302 train steps 105822
Length 181
Episode 91831 total samples 3398883 train steps 105903
Length 192
Episode 91855 total samples 3401591 train steps 105987
Length 497
Episode 91886 total samples 3404162 train steps 106068
Length 319
Episode 91909 total samples 3406731 train steps 106148
Length 145
exploration 0.0999991000096
Episode 91934 total samples 3409493 train steps 106234
Length 703
Episode 91958 total samples 3412239 train steps 106320
Length 700
Episode 91983 total samples 3414770 train steps 106399
Length 185
Episode 91998 total samples 3417423 train steps 106482
Length 935
Episode 92026 total samples 3419926 train steps 106560
Length 184
Episode 92056 total samples 3422679 train steps 106646
Length 1031
Episode 92078 total samples 3425515 train steps 106735
Length 794
Episode 92099 total samples 3428124 train steps 106816
Length 313
Episode 92121 total samples 3430679 train steps 106896
Length 439
Episode 92151 total samples 3433189 train steps 106975
Length 280
exploration 0.0999991000096
Episode 92178 total samples 3435692 train steps 107053
Length 587
Episode 92195 total samples 3438469 train steps 107140
Length 640
Episode 92214 total samples 3441559 train steps 107236
Length 1641
Episode 92235 total samples 3444109 train steps 107316
Length 104
Episode 92256 total samples 3446742 train steps 107398
Length 299
Episode 92274 total samples 3449256 train steps 107477
Length 343
Episode 92294 total samples 3451768 train steps 107555
Length 66
Episode 92326 total samples 3454365 train steps 107636
Length 225
Episode 92345 total samples 3456907 train steps 107716
Length 467
exploration 0.0999991000096
Episode 92366 total samples 3459585 train steps 107800
Length 1245
Episode 92389 total samples 3462111 train steps 107878
Length 776
Episode 92416 total samples 3464654 train steps 107958
Length 492
Episode 92439 total samples 3467161 train steps 108036
Length 68
Episode 92457 total samples 3469764 train steps 108118
Length 236
Episode 92468 total samples 3472265 train steps 108196
Length 60
Episode 92498 total samples 3474820 train steps 108276
Length 268
Episode 92515 total samples 3477358 train steps 108355
Length 100
Episode 92536 total samples 3480084 train steps 108440
Length 1514
Episode 92548 total samples 3482620 train steps 108519
Length 719
exploration 0.0999991000096
Episode 92561 total samples 3485911 train steps 108622
Length 1610
Episode 92597 total samples 3488572 train steps 108705
Length 338
Episode 92620 total samples 3491253 train steps 108789
Length 500
Episode 92636 total samples 3493776 train steps 108868
Length 178
Episode 92655 total samples 3496729 train steps 108960
Length 1495
Episode 92669 total samples 3499230 train steps 109038
Length 175
Episode 92690 total samples 3501745 train steps 109117
Length 597
Episode 92716 total samples 3504356 train steps 109199
Length 243
Episode 92735 total samples 3506870 train steps 109277
Length 186
exploration 0.0999991000096
Episode 92749 total samples 3509389 train steps 109356
Length 99
Episode 92770 total samples 3512055 train steps 109439
Length 442
Episode 92789 total samples 3514558 train steps 109517
Length 661
Episode 92807 total samples 3517137 train steps 109598
Length 170
Episode 92846 total samples 3519676 train steps 109677
Length 110
Episode 92867 total samples 3522312 train steps 109760
Length 622
Episode 92907 total samples 3524964 train steps 109843
Length 368
Episode 92930 total samples 3527505 train steps 109922
Length 274
Episode 92949 total samples 3530053 train steps 110002
Length 320
Episode 92980 total samples 3532636 train steps 110082
Length 400
exploration 0.0999991000096
Episode 93011 total samples 3535158 train steps 110161
Length 348
Episode 93033 total samples 3537661 train steps 110239
Length 58
Episode 93052 total samples 3540665 train steps 110333
Length 1450
Episode 93084 total samples 3543219 train steps 110413
Length 253
Episode 93097 total samples 3545984 train steps 110499
Length 871
Episode 93114 total samples 3548570 train steps 110580
Length 180
Episode 93136 total samples 3551329 train steps 110667
Length 757
Episode 93213 total samples 3553992 train steps 110750
Length 398
Episode 93234 total samples 3556506 train steps 110828
Length 53
Episode 93245 total samples 3559010 train steps 110907
Length 544
exploration 0.0999991000096
Episode 93264 total samples 3561531 train steps 110985
Length 62
Episode 93286 total samples 3564141 train steps 111067
Length 242
Episode 93304 total samples 3566657 train steps 111146
Length 212
Episode 93324 total samples 3569178 train steps 111224
Length 130
Episode 93345 total samples 3571826 train steps 111307
Length 306
Episode 93367 total samples 3574427 train steps 111388
Length 261
Episode 93387 total samples 3576971 train steps 111468
Length 482
Episode 93414 total samples 3579500 train steps 111547
Length 76
Episode 93440 total samples 3582334 train steps 111635
Length 918
exploration 0.0999991000096
Episode 93465 total samples 3584931 train steps 111717
Length 247
Episode 93493 total samples 3587548 train steps 111798
Length 582
Episode 93516 total samples 3590103 train steps 111878
Length 251
Episode 93550 total samples 3592671 train steps 111958
Length 272
Episode 93570 total samples 3595189 train steps 112037
Length 120
Episode 93594 total samples 3597697 train steps 112116
Length 518
Episode 93640 total samples 3600221 train steps 112194
Length 150
Episode 93661 total samples 3602768 train steps 112274
Length 378
Episode 93684 total samples 3605288 train steps 112353
Length 80
Episode 93702 total samples 3607901 train steps 112434
Length 348
exploration 0.0999991000096
Episode 93728 total samples 3610440 train steps 112514
Length 77
Episode 93743 total samples 3612970 train steps 112593
Length 182
Episode 93769 total samples 3615511 train steps 112672
Length 261
Episode 93791 total samples 3618343 train steps 112761
Length 671
Episode 93819 total samples 3621173 train steps 112849
Length 804
Episode 93842 total samples 3623678 train steps 112927
Length 168
Episode 93871 total samples 3626335 train steps 113010
Length 391
Episode 93892 total samples 3628910 train steps 113091
Length 378
Episode 93922 total samples 3631543 train steps 113173
Length 371
Episode 93942 total samples 3634422 train steps 113263
Length 1642
exploration 0.0999991000096
Episode 93973 total samples 3637165 train steps 113349
Length 629
Episode 93990 total samples 3639769 train steps 113430
Length 387
Episode 94008 total samples 3642436 train steps 113514
Length 581
Episode 94038 total samples 3644962 train steps 113593
Length 109
Episode 94056 total samples 3647555 train steps 113674
Length 922
Episode 94091 total samples 3650300 train steps 113759
Length 1085
Episode 94117 total samples 3652839 train steps 113839
Length 128
Episode 94134 total samples 3655346 train steps 113917
Length 414
Episode 94159 total samples 3657901 train steps 113997
Length 241
exploration 0.0999991000096
Episode 94179 total samples 3660418 train steps 114076
Length 670
Episode 94198 total samples 3662964 train steps 114155
Length 113
Episode 94216 total samples 3665572 train steps 114237
Length 220
Episode 94231 total samples 3668088 train steps 114315
Length 219
Episode 94253 total samples 3670659 train steps 114396
Length 491
Episode 94278 total samples 3673234 train steps 114476
Length 205
Episode 94295 total samples 3675738 train steps 114554
Length 19
Episode 94322 total samples 3678403 train steps 114638
Length 365
Episode 94349 total samples 3681259 train steps 114727
Length 862
Episode 94363 total samples 3683807 train steps 114806
Length 809
exploration 0.0999991000096
Episode 94380 total samples 3686557 train steps 114892
Length 556
Episode 94400 total samples 3689080 train steps 114971
Length 277
Episode 94680 total samples 3691581 train steps 115049
Length 9
Episode 95029 total samples 3694084 train steps 115128
Length 27
Episode 95295 total samples 3696596 train steps 115206
Length 48
Episode 95516 total samples 3699108 train steps 115285
Length 27
Episode 95772 total samples 3701613 train steps 115363
Length 24
Episode 96057 total samples 3704117 train steps 115441
Length 14
Episode 96317 total samples 3706622 train steps 115519
Length 12
Episode 96549 total samples 3709129 train steps 115598
Length 32
exploration 0.0999991000096
Episode 96773 total samples 3711630 train steps 115676
Length 21
Episode 97037 total samples 3714139 train steps 115754
Length 42
Episode 97310 total samples 3716647 train steps 115833
Length 35
Episode 97496 total samples 3719161 train steps 115911
Length 41
Episode 97712 total samples 3721669 train steps 115990
Length 34
Episode 97895 total samples 3724184 train steps 116068
Length 42
Episode 98147 total samples 3726690 train steps 116147
Length 14
Episode 98320 total samples 3729210 train steps 116225
Length 55
Episode 98506 total samples 3731717 train steps 116304
Length 43
Episode 98686 total samples 3734220 train steps 116382
Length 12
exploration 0.0999991000096
Episode 98835 total samples 3736727 train steps 116460
Length 20
Episode 99042 total samples 3739231 train steps 116538
Length 10
Episode 99241 total samples 3741733 train steps 116617
Length 20
Episode 99432 total samples 3744239 train steps 116695
Length 22
Episode 99591 total samples 3746747 train steps 116773
Length 17
Episode 99777 total samples 3749250 train steps 116852
Length 35
Episode 99963 total samples 3751758 train steps 116930
Length 37
Episode 100154 total samples 3754275 train steps 117009
Length 36
Episode 100402 total samples 3756789 train steps 117087
Length 49
Episode 100662 total samples 3759290 train steps 117165
Length 13
exploration 0.0999991000096
Episode 100822 total samples 3761809 train steps 117244
Length 56
Episode 100950 total samples 3764327 train steps 117323
Length 85
Episode 101111 total samples 3766846 train steps 117401
Length 43
Episode 101243 total samples 3769352 train steps 117480
Length 24
Episode 101366 total samples 3771857 train steps 117558
Length 49
Episode 101513 total samples 3774420 train steps 117638
Length 136
Episode 101624 total samples 3776931 train steps 117717
Length 34
Episode 101717 total samples 3779453 train steps 117795
Length 47
Episode 101801 total samples 3782023 train steps 117876
Length 186
Episode 101905 total samples 3784525 train steps 117954
Length 15
exploration 0.0999991000096
Episode 101992 total samples 3787045 train steps 118033
Length 64
Episode 102067 total samples 3789553 train steps 118111
Length 25
Episode 102136 total samples 3792092 train steps 118190
Length 87
Episode 102208 total samples 3794604 train steps 118269
Length 93
Episode 102274 total samples 3797145 train steps 118348
Length 147
Episode 102341 total samples 3799646 train steps 118426
Length 77
Episode 102393 total samples 3802175 train steps 118505
Length 92
Episode 102460 total samples 3804679 train steps 118584
Length 96
Episode 102526 total samples 3807218 train steps 118663
Length 125
Episode 102585 total samples 3809726 train steps 118741
Length 120
exploration 0.0999991000096
Episode 102655 total samples 3812272 train steps 118821
Length 136
Episode 102703 total samples 3814788 train steps 118900
Length 400
Episode 102749 total samples 3817311 train steps 118978
Length 63
Episode 102797 total samples 3819859 train steps 119058
Length 118
Episode 102851 total samples 3822375 train steps 119137
Length 44
Episode 102902 total samples 3824919 train steps 119216
Length 99
Episode 102956 total samples 3827453 train steps 119295
Length 141
Episode 103006 total samples 3829987 train steps 119375
Length 98
Episode 103057 total samples 3832509 train steps 119453
Length 65
Episode 103094 total samples 3835013 train steps 119532
Length 683
exploration 0.0999991000096
Episode 103133 total samples 3837533 train steps 119610
Length 169
Episode 103176 total samples 3840073 train steps 119690
Length 114
Episode 103225 total samples 3842624 train steps 119769
Length 443
Episode 103272 total samples 3845208 train steps 119850
Length 179
Episode 103307 total samples 3847781 train steps 119931
Length 874
Episode 103340 total samples 3850288 train steps 120009
Length 139
Episode 103373 total samples 3852842 train steps 120089
Length 286
Episode 103405 total samples 3855344 train steps 120167
Length 259
Episode 103441 total samples 3857848 train steps 120245
Length 146
Episode 103481 total samples 3860360 train steps 120324
Length 126
exploration 0.0999991000096
Episode 103521 total samples 3862988 train steps 120406
Length 310
Episode 103559 total samples 3865800 train steps 120494
Length 803
Episode 103601 total samples 3868422 train steps 120576
Length 258
Episode 103638 total samples 3870977 train steps 120656
Length 163
Episode 103663 total samples 3873542 train steps 120736
Length 134
Episode 103700 total samples 3876111 train steps 120816
Length 167
Episode 103733 total samples 3878743 train steps 120898
Length 463
Episode 103758 total samples 3881253 train steps 120977
Length 198
Episode 103787 total samples 3883933 train steps 121060
Length 408
exploration 0.0999991000096
Episode 103822 total samples 3886623 train steps 121144
Length 421
Episode 103854 total samples 3889227 train steps 121226
Length 218
Episode 103880 total samples 3891801 train steps 121306
Length 332
Episode 103911 total samples 3894325 train steps 121385
Length 117
Episode 103943 total samples 3896876 train steps 121465
Length 636
Episode 103975 total samples 3899453 train steps 121545
Length 205
Episode 104009 total samples 3902029 train steps 121626
Length 255
Episode 104033 total samples 3904613 train steps 121707
Length 208
Episode 104063 total samples 3907273 train steps 121790
Length 401
Episode 104101 total samples 3909845 train steps 121870
Length 159
exploration 0.0999991000096
Episode 104138 total samples 3912374 train steps 121949
Length 94
Episode 104162 total samples 3914925 train steps 122029
Length 444
Episode 104191 total samples 3917445 train steps 122108
Length 159
Episode 104215 total samples 3919966 train steps 122186
Length 132
Episode 104235 total samples 3922600 train steps 122269
Length 496
Episode 104265 total samples 3925275 train steps 122352
Length 393
Episode 104293 total samples 3927867 train steps 122433
Length 193
Episode 104331 total samples 3930382 train steps 122512
Length 125
Episode 104363 total samples 3932900 train steps 122591
Length 244
Episode 104400 total samples 3935425 train steps 122670
Length 188
exploration 0.0999991000096
Episode 104422 total samples 3937955 train steps 122749
Length 397
Episode 104446 total samples 3940487 train steps 122828
Length 124
Episode 104482 total samples 3943129 train steps 122910
Length 477
Episode 104512 total samples 3945660 train steps 122989
Length 192
Episode 104542 total samples 3948353 train steps 123074
Length 454
Episode 104571 total samples 3950891 train steps 123153
Length 180
Episode 104612 total samples 3953428 train steps 123232
Length 185
Episode 104636 total samples 3955965 train steps 123311
Length 398
Episode 104670 total samples 3958505 train steps 123391
Length 96
exploration 0.0999991000096
Episode 104700 total samples 3961061 train steps 123471
Length 360
Episode 104734 total samples 3963566 train steps 123549
Length 89
Episode 104761 total samples 3966185 train steps 123631
Length 410
Episode 104782 total samples 3968811 train steps 123713
Length 261
Episode 104809 total samples 3971655 train steps 123802
Length 809
Episode 104831 total samples 3974173 train steps 123880
Length 148
Episode 104858 total samples 3976823 train steps 123963
Length 315
Episode 104886 total samples 3979369 train steps 124043
Length 380
Episode 104913 total samples 3981870 train steps 124121
Length 145
Episode 104943 total samples 3984457 train steps 124202
Length 763
exploration 0.0999991000096
Episode 104985 total samples 3987012 train steps 124282
Length 135
Episode 105012 total samples 3989532 train steps 124360
Length 367
Episode 105045 total samples 3992132 train steps 124442
Length 240
Episode 105065 total samples 3994699 train steps 124522
Length 372
Episode 105089 total samples 3997305 train steps 124603
Length 453
Episode 105112 total samples 3999868 train steps 124683
Length 154
Episode 105140 total samples 4002370 train steps 124762
Length 158
Episode 105163 total samples 4004983 train steps 124843
Length 594
Episode 105182 total samples 4007494 train steps 124922
Length 353
Episode 105206 total samples 4010118 train steps 125004
Length 321
exploration 0.0999991000096
Episode 105242 total samples 4012625 train steps 125082
Length 608
Episode 105266 total samples 4015151 train steps 125161
Length 229
Episode 105304 total samples 4017683 train steps 125240
Length 141
Episode 105345 total samples 4020234 train steps 125320
Length 125
Episode 105367 total samples 4022860 train steps 125402
Length 291
Episode 105391 total samples 4025403 train steps 125481
Length 108
Episode 105421 total samples 4027926 train steps 125560
Length 167
Episode 105453 total samples 4030447 train steps 125639
Length 207
Episode 105483 total samples 4033172 train steps 125724
Length 506
Episode 105512 total samples 4035707 train steps 125803
Length 165
exploration 0.0999991000096
Episode 105535 total samples 4038326 train steps 125885
Length 264
Episode 105560 total samples 4040916 train steps 125966
Length 289
Episode 105581 total samples 4043421 train steps 126044
Length 266
Episode 105605 total samples 4045984 train steps 126124
Length 432
Episode 105620 total samples 4049109 train steps 126222
Length 1270
Episode 105641 total samples 4051625 train steps 126301
Length 78
Episode 105660 total samples 4054169 train steps 126380
Length 173
Episode 105691 total samples 4056683 train steps 126459
Length 158
Episode 105719 total samples 4059490 train steps 126547
Length 652
exploration 0.0999991000096
Episode 105749 total samples 4062056 train steps 126627
Length 381
Episode 105772 total samples 4064730 train steps 126710
Length 357
Episode 105797 total samples 4067309 train steps 126791
Length 271
Episode 105829 total samples 4070115 train steps 126879
Length 677
Episode 105857 total samples 4072908 train steps 126966
Length 725
Episode 105884 total samples 4075418 train steps 127044
Length 29
Episode 105909 total samples 4077957 train steps 127124
Length 263
Episode 105943 total samples 4080593 train steps 127206
Length 294
Episode 105977 total samples 4083166 train steps 127286
Length 383
Episode 106001 total samples 4085700 train steps 127366
Length 224
exploration 0.0999991000096
Episode 106024 total samples 4088273 train steps 127446
Length 200
Episode 106051 total samples 4090838 train steps 127526
Length 211
Episode 106067 total samples 4093343 train steps 127604
Length 162
Episode 106086 total samples 4096101 train steps 127691
Length 570
Episode 106107 total samples 4098620 train steps 127769
Length 138
Episode 106136 total samples 4101209 train steps 127850
Length 218
Episode 106163 total samples 4103763 train steps 127930
Length 404
Episode 106187 total samples 4106283 train steps 128009
Length 78
Episode 106209 total samples 4108801 train steps 128088
Length 263
Episode 106229 total samples 4111373 train steps 128168
Length 280
exploration 0.0999991000096
Episode 106256 total samples 4113880 train steps 128246
Length 15
Episode 106282 total samples 4116444 train steps 128326
Length 451
Episode 106306 total samples 4119069 train steps 128408
Length 383
Episode 106326 total samples 4121594 train steps 128487
Length 79
Episode 106351 total samples 4124221 train steps 128569
Length 386
Episode 106377 total samples 4126733 train steps 128648
Length 64
Episode 106397 total samples 4129246 train steps 128726
Length 113
Episode 106417 total samples 4131986 train steps 128812
Length 623
Episode 106440 total samples 4134534 train steps 128892
Length 427
exploration 0.0999991000096
Episode 106466 total samples 4137045 train steps 128970
Length 44
Episode 106493 total samples 4139548 train steps 129048
Length 138
Episode 106561 total samples 4142104 train steps 129128
Length 244
Episode 106585 total samples 4144637 train steps 129207
Length 314
Episode 106604 total samples 4147185 train steps 129287
Length 115
Episode 106627 total samples 4149771 train steps 129368
Length 180
Episode 106654 total samples 4152473 train steps 129452
Length 431
Episode 106673 total samples 4155152 train steps 129536
Length 357
Episode 106687 total samples 4157678 train steps 129615
Length 634
Episode 106705 total samples 4160237 train steps 129695
Length 189
exploration 0.0999991000096
Episode 106738 total samples 4162742 train steps 129773
Length 88
Episode 106756 total samples 4165272 train steps 129852
Length 80
Episode 106780 total samples 4167829 train steps 129932
Length 406
Episode 106799 total samples 4170375 train steps 130012
Length 334
Episode 106836 total samples 4172891 train steps 130090
Length 237
Episode 106862 total samples 4175447 train steps 130170
Length 626
Episode 106900 total samples 4177966 train steps 130249
Length 178
Episode 106923 total samples 4180489 train steps 130328
Length 284
Episode 106942 total samples 4183135 train steps 130410
Length 316
Episode 106957 total samples 4185722 train steps 130491
Length 281
exploration 0.0999991000096
Episode 106982 total samples 4188372 train steps 130574
Length 425
Episode 107005 total samples 4191177 train steps 130662
Length 904
Episode 107036 total samples 4193905 train steps 130747
Length 662
Episode 107056 total samples 4196413 train steps 130825
Length 230
Episode 107072 total samples 4199422 train steps 130919
Length 1085
Episode 107089 total samples 4202131 train steps 131004
Length 1110
Episode 107108 total samples 4204719 train steps 131085
Length 238
Episode 107145 total samples 4207339 train steps 131167
Length 348
Episode 107171 total samples 4210116 train steps 131254
Length 886
exploration 0.0999991000096
Episode 107182 total samples 4212699 train steps 131334
Length 725
Episode 107201 total samples 4215241 train steps 131414
Length 191
Episode 107227 total samples 4217806 train steps 131494
Length 453
Episode 107246 total samples 4220850 train steps 131589
Length 1387
Episode 107269 total samples 4223484 train steps 131671
Length 616
Episode 107283 total samples 4226025 train steps 131751
Length 360
Episode 107323 total samples 4228547 train steps 131830
Length 87
Episode 107346 total samples 4231080 train steps 131909
Length 448
Episode 107398 total samples 4233582 train steps 131987
Length 91
Episode 107419 total samples 4236117 train steps 132066
Length 634
exploration 0.0999991000096
Episode 107450 total samples 4238620 train steps 132144
Length 79
Episode 107482 total samples 4241184 train steps 132224
Length 168
Episode 107506 total samples 4243731 train steps 132304
Length 313
Episode 107522 total samples 4246322 train steps 132385
Length 356
Episode 107549 total samples 4248843 train steps 132464
Length 113
Episode 107569 total samples 4251398 train steps 132544
Length 226
Episode 107589 total samples 4253932 train steps 132623
Length 187
Episode 107724 total samples 4256517 train steps 132704
Length 202
Episode 107750 total samples 4259081 train steps 132784
Length 378
Episode 107771 total samples 4261704 train steps 132866
Length 318
exploration 0.0999991000096
Episode 107788 total samples 4264340 train steps 132948
Length 517
Episode 107804 total samples 4266843 train steps 133026
Length 425
Episode 107836 total samples 4269354 train steps 133105
Length 184
Episode 107849 total samples 4271918 train steps 133185
Length 212
Episode 107868 total samples 4274661 train steps 133271
Length 968
Episode 107887 total samples 4277215 train steps 133350
Length 327
Episode 107911 total samples 4279779 train steps 133431
Length 686
Episode 107930 total samples 4282785 train steps 133525
Length 1532
Episode 107949 total samples 4285415 train steps 133607
Length 309
exploration 0.0999991000096
Episode 107970 total samples 4287957 train steps 133686
Length 501
Episode 107997 total samples 4290836 train steps 133776
Length 770
Episode 108028 total samples 4293746 train steps 133867
Length 929
Episode 108043 total samples 4296341 train steps 133948
Length 529
Episode 108066 total samples 4299151 train steps 134036
Length 762
Episode 108081 total samples 4301661 train steps 134114
Length 415
Episode 108096 total samples 4304252 train steps 134195
Length 806
Episode 108116 total samples 4306844 train steps 134276
Length 353
Episode 108130 total samples 4309477 train steps 134359
Length 283
Episode 108141 total samples 4312354 train steps 134449
Length 1172
exploration 0.0999991000096
Episode 108154 total samples 4315086 train steps 134534
Length 582
Episode 108177 total samples 4317683 train steps 134615
Length 566
Episode 108206 total samples 4320214 train steps 134694
Length 290
Episode 108222 total samples 4322730 train steps 134773
Length 232
Episode 108249 total samples 4325233 train steps 134851
Length 147
Episode 108268 total samples 4327844 train steps 134933
Length 229
Episode 108293 total samples 4330345 train steps 135011
Length 352
Episode 108311 total samples 4332865 train steps 135090
Length 113
Episode 108334 total samples 4335427 train steps 135170
Length 827
exploration 0.0999991000096
Episode 108355 total samples 4338527 train steps 135266
Length 1250
Episode 108376 total samples 4341093 train steps 135347
Length 139
Episode 108391 total samples 4343594 train steps 135425
Length 177
Episode 108407 total samples 4346203 train steps 135506
Length 269
Episode 108430 total samples 4348814 train steps 135588
Length 297
Episode 108518 total samples 4351336 train steps 135667
Length 73
Episode 108555 total samples 4353894 train steps 135747
Length 428
Episode 108582 total samples 4356599 train steps 135831
Length 440
Episode 108598 total samples 4359138 train steps 135911
Length 472
Episode 108616 total samples 4361706 train steps 135991
Length 378
exploration 0.0999991000096
Episode 108643 total samples 4364327 train steps 136073
Length 349
Episode 108664 total samples 4366875 train steps 136152
Length 439
Episode 108682 total samples 4369515 train steps 136235
Length 392
Episode 108700 total samples 4372049 train steps 136314
Length 203
Episode 108721 total samples 4374556 train steps 136392
Length 150
Episode 108744 total samples 4377058 train steps 136471
Length 33
Episode 108768 total samples 4379890 train steps 136559
Length 674
Episode 108783 total samples 4382401 train steps 136638
Length 312
Episode 108806 total samples 4385060 train steps 136721
Length 335
Episode 108819 total samples 4387899 train steps 136809
Length 1491
exploration 0.0999991000096
Episode 108839 total samples 4390543 train steps 136892
Length 424
Episode 108858 total samples 4393065 train steps 136971
Length 228
Episode 108875 total samples 4395643 train steps 137051
Length 676
Episode 108889 total samples 4398221 train steps 137132
Length 823
Episode 108902 total samples 4400857 train steps 137214
Length 1094
Episode 108928 total samples 4403598 train steps 137300
Length 722
Episode 108944 total samples 4406151 train steps 137380
Length 200
Episode 108969 total samples 4408717 train steps 137460
Length 136
Episode 108984 total samples 4411451 train steps 137545
Length 518
exploration 0.0999991000096
Episode 109005 total samples 4414096 train steps 137628
Length 463
Episode 109020 total samples 4416626 train steps 137707
Length 266
Episode 109032 total samples 4419467 train steps 137796
Length 704
Episode 109051 total samples 4421983 train steps 137874
Length 87
Episode 109076 total samples 4424550 train steps 137955
Length 159
Episode 109096 total samples 4427152 train steps 138036
Length 225
Episode 109113 total samples 4429804 train steps 138119
Length 715
Episode 109132 total samples 4432308 train steps 138197
Length 58
Episode 109147 total samples 4434884 train steps 138278
Length 333
Episode 109157 total samples 4437626 train steps 138363
Length 1172
exploration 0.0999991000096
Episode 109183 total samples 4440358 train steps 138449
Length 587
Episode 109201 total samples 4443145 train steps 138536
Length 1032
Episode 109222 total samples 4446223 train steps 138632
Length 1402
Episode 109245 total samples 4448958 train steps 138717
Length 502
Episode 109261 total samples 4451472 train steps 138796
Length 245
Episode 109283 total samples 4454059 train steps 138877
Length 323
Episode 109302 total samples 4456576 train steps 138955
Length 69
Episode 109336 total samples 4459163 train steps 139036
Length 184
Episode 109360 total samples 4461874 train steps 139121
Length 481
exploration 0.0999991000096
Episode 109375 total samples 4464498 train steps 139203
Length 282
Episode 109386 total samples 4467230 train steps 139288
Length 927
Episode 109411 total samples 4469907 train steps 139372
Length 533
Episode 109432 total samples 4472664 train steps 139458
Length 927
Episode 109446 total samples 4475207 train steps 139538
Length 1083
Episode 109462 total samples 4477890 train steps 139622
Length 1164
Episode 109479 total samples 4480401 train steps 139700
Length 155
Episode 109491 total samples 4483023 train steps 139782
Length 265
Episode 109511 total samples 4485614 train steps 139863
Length 188
Episode 109531 total samples 4488469 train steps 139952
Length 818
exploration 0.0999991000096
Episode 109558 total samples 4491092 train steps 140034
Length 1037
Episode 109581 total samples 4493727 train steps 140116
Length 285
Episode 109595 total samples 4496284 train steps 140196
Length 670
Episode 109613 total samples 4498860 train steps 140277
Length 497
Episode 109638 total samples 4501438 train steps 140357
Length 164
Episode 109682 total samples 4504141 train steps 140442
Length 460
Episode 109698 total samples 4506855 train steps 140527
Length 880
Episode 109721 total samples 4509424 train steps 140607
Length 152
Episode 109738 total samples 4511938 train steps 140686
Length 663
exploration 0.0999991000096
Episode 109765 total samples 4514765 train steps 140774
Length 659
Episode 109790 total samples 4517330 train steps 140854
Length 148
Episode 109804 total samples 4519895 train steps 140934
Length 167
Episode 109824 total samples 4522543 train steps 141017
Length 470
Episode 109869 total samples 4525203 train steps 141100
Length 488
Episode 109883 total samples 4527851 train steps 141183
Length 385
Episode 109912 total samples 4530378 train steps 141262
Length 87
Episode 109931 total samples 4532889 train steps 141340
Length 50
Episode 109955 total samples 4535396 train steps 141419
Length 69
Episode 109974 total samples 4538002 train steps 141500
Length 229
exploration 0.0999991000096
Episode 109997 total samples 4540516 train steps 141579
Length 609
Episode 110012 total samples 4543226 train steps 141663
Length 871
Episode 110023 total samples 4545776 train steps 141743
Length 413
Episode 110055 total samples 4548303 train steps 141822
Length 110
Episode 110073 total samples 4550990 train steps 141906
Length 467
Episode 110092 total samples 4553530 train steps 141985
Length 298
Episode 110111 total samples 4556164 train steps 142068
Length 399
Episode 110135 total samples 4558686 train steps 142146
Length 185
Episode 110158 total samples 4561657 train steps 142239
Length 1009
Episode 110171 total samples 4564199 train steps 142319
Length 537
exploration 0.0999991000096
Episode 110202 total samples 4566755 train steps 142399
Length 287
Episode 110218 total samples 4569265 train steps 142477
Length 122
Episode 110239 total samples 4571784 train steps 142556
Length 382
Episode 110265 total samples 4574390 train steps 142637
Length 219
Episode 110279 total samples 4577015 train steps 142719
Length 940
Episode 110301 total samples 4579527 train steps 142798
Length 284
Episode 110323 total samples 4582138 train steps 142879
Length 443
Episode 110342 total samples 4584660 train steps 142958
Length 613
Episode 110355 total samples 4587236 train steps 143039
Length 546
exploration 0.0999991000096
Episode 110374 total samples 4589792 train steps 143118
Length 238
Episode 110395 total samples 4592300 train steps 143197
Length 155
Episode 110418 total samples 4594805 train steps 143275
Length 78
Episode 110439 total samples 4597336 train steps 143354
Length 432
Episode 110467 total samples 4599838 train steps 143432
Length 397
Episode 110489 total samples 4602627 train steps 143520
Length 655
Episode 110509 total samples 4605185 train steps 143600
Length 345
Episode 110534 total samples 4607689 train steps 143678
Length 365
Episode 110545 total samples 4610383 train steps 143762
Length 996
Episode 110588 total samples 4612898 train steps 143841
Length 126
exploration 0.0999991000096
Episode 110607 total samples 4615473 train steps 143921
Length 234
Episode 110632 total samples 4618042 train steps 144001
Length 181
Episode 110654 total samples 4620631 train steps 144082
Length 474
Episode 110678 total samples 4623647 train steps 144176
Length 1205
Episode 110694 total samples 4626193 train steps 144256
Length 204
Episode 110715 total samples 4628857 train steps 144339
Length 648
Episode 110732 total samples 4631679 train steps 144427
Length 847
Episode 110750 total samples 4634183 train steps 144506
Length 33
Episode 110783 total samples 4636746 train steps 144586
Length 231
Episode 110805 total samples 4639507 train steps 144672
Length 565
exploration 0.0999991000096
Episode 110828 total samples 4642149 train steps 144755
Length 354
Episode 110852 total samples 4644712 train steps 144835
Length 149
Episode 110873 total samples 4647396 train steps 144919
Length 924
Episode 110890 total samples 4649999 train steps 145000
Length 240
Episode 110917 total samples 4652645 train steps 145083
Length 519
Episode 110931 total samples 4655226 train steps 145163
Length 377
Episode 110950 total samples 4657760 train steps 145242
Length 235
Episode 110966 total samples 4660277 train steps 145321
Length 160
Episode 110989 total samples 4663203 train steps 145413
Length 1130
exploration 0.0999991000096
Episode 111003 total samples 4665744 train steps 145492
Length 728
Episode 111028 total samples 4668623 train steps 145582
Length 1355
Episode 111043 total samples 4671324 train steps 145666
Length 630
Episode 111061 total samples 4673946 train steps 145748
Length 652
Episode 111081 total samples 4676566 train steps 145830
Length 287
Episode 111094 total samples 4679295 train steps 145915
Length 479
Episode 111116 total samples 4681801 train steps 145994
Length 273
Episode 111134 total samples 4684303 train steps 146072
Length 268
Episode 111154 total samples 4686841 train steps 146151
Length 256
Episode 111178 total samples 4689455 train steps 146233
Length 378
exploration 0.0999991000096
Episode 111197 total samples 4692371 train steps 146324
Length 1121
Episode 111219 total samples 4694964 train steps 146405
Length 451
Episode 111240 total samples 4697495 train steps 146484
Length 1013
Episode 111252 total samples 4700494 train steps 146578
Length 1117
Episode 111279 total samples 4703048 train steps 146658
Length 155
Episode 111430 total samples 4705579 train steps 146737
Length 91
Episode 111452 total samples 4708258 train steps 146821
Length 449
Episode 111464 total samples 4710883 train steps 146903
Length 593
Episode 111475 total samples 4713458 train steps 146983
Length 764
exploration 0.0999991000096
Episode 111499 total samples 4716105 train steps 147066
Length 316
Episode 111517 total samples 4718764 train steps 147149
Length 362
Episode 111541 total samples 4721386 train steps 147231
Length 286
Episode 111556 total samples 4723944 train steps 147311
Length 213
Episode 111588 total samples 4726502 train steps 147391
Length 138
Episode 111604 total samples 4729024 train steps 147469
Length 349
Episode 111620 total samples 4731588 train steps 147550
Length 267
Episode 111642 total samples 4734198 train steps 147631
Length 267
Episode 111656 total samples 4736738 train steps 147711
Length 179
Episode 111679 total samples 4739439 train steps 147795
Length 653
exploration 0.0999991000096
Episode 111700 total samples 4742121 train steps 147879
Length 534
Episode 111727 total samples 4744628 train steps 147957
Length 126
Episode 111747 total samples 4747216 train steps 148038
Length 248
Episode 111776 total samples 4749820 train steps 148119
Length 381
Episode 111806 total samples 4752414 train steps 148200
Length 296
Episode 111831 total samples 4754961 train steps 148280
Length 123
Episode 111852 total samples 4757505 train steps 148360
Length 281
Episode 111878 total samples 4760018 train steps 148438
Length 187
Episode 111897 total samples 4762917 train steps 148529
Length 1337
exploration 0.0999991000096
Episode 111917 total samples 4765519 train steps 148610
Length 529
Episode 111934 total samples 4768122 train steps 148691
Length 476
Episode 111955 total samples 4770702 train steps 148772
Length 480
Episode 111976 total samples 4773231 train steps 148851
Length 861
Episode 112002 total samples 4775794 train steps 148931
Length 309
Episode 112020 total samples 4778404 train steps 149013
Length 234
Episode 112046 total samples 4781065 train steps 149096
Length 451
Episode 112060 total samples 4783869 train steps 149183
Length 1070
Episode 112160 total samples 4786373 train steps 149262
Length 31
Episode 112191 total samples 4788888 train steps 149340
Length 50
exploration 0.0999991000096
Episode 112213 total samples 4791404 train steps 149419
Length 763
Episode 112235 total samples 4793954 train steps 149499
Length 504
Episode 112252 total samples 4796484 train steps 149578
Length 168
Episode 112269 total samples 4799102 train steps 149659
Length 370
Episode 112292 total samples 4801638 train steps 149739
Length 346
Episode 112313 total samples 4804185 train steps 149818
Length 488
Episode 112334 total samples 4806819 train steps 149901
Length 603
Episode 112363 total samples 4809518 train steps 149985
Length 767
Episode 112387 total samples 4812133 train steps 150067
Length 291
Episode 112430 total samples 4814800 train steps 150150
Length 505
exploration 0.0999991000096
Episode 112448 total samples 4817306 train steps 150228
Length 185
Episode 112466 total samples 4819808 train steps 150306
Length 224
Episode 112483 total samples 4822415 train steps 150388
Length 384
Episode 112505 total samples 4824926 train steps 150466
Length 61
Episode 112519 total samples 4827543 train steps 150548
Length 241
Episode 112534 total samples 4830095 train steps 150628
Length 447
Episode 112561 total samples 4832788 train steps 150712
Length 603
Episode 112586 total samples 4835438 train steps 150795
Length 329
Episode 112603 total samples 4837986 train steps 150875
Length 733
Episode 112629 total samples 4840538 train steps 150954
Length 365
exploration 0.0999991000096
Episode 112648 total samples 4843360 train steps 151042
Length 671
Episode 112667 total samples 4846019 train steps 151126
Length 336
Episode 112691 total samples 4848765 train steps 151211
Length 849
Episode 112709 total samples 4851295 train steps 151290
Length 220
Episode 112725 total samples 4853858 train steps 151371
Length 568
Episode 112739 total samples 4856604 train steps 151456
Length 1317
Episode 112846 total samples 4859107 train steps 151535
Length 8
Episode 112925 total samples 4861634 train steps 151614
Length 100
Episode 112946 total samples 4864156 train steps 151692
Length 64
exploration 0.0999991000096
Episode 112964 total samples 4866735 train steps 151773
Length 169
Episode 112984 total samples 4869258 train steps 151852
Length 392
Episode 113000 total samples 4871874 train steps 151934
Length 339
Episode 113023 total samples 4874557 train steps 152017
Length 1206
Episode 113045 total samples 4877104 train steps 152097
Length 157
Episode 113068 total samples 4879693 train steps 152178
Length 847
Episode 113087 total samples 4882254 train steps 152258
Length 491
Episode 113109 total samples 4884787 train steps 152337
Length 137
Episode 113126 total samples 4887523 train steps 152423
Length 830
Episode 113141 total samples 4890238 train steps 152507
Length 1054
exploration 0.0999991000096
Episode 113172 total samples 4892806 train steps 152588
Length 431
Episode 113194 total samples 4895332 train steps 152667
Length 96
Episode 113220 total samples 4897834 train steps 152745
Length 158
Episode 113238 total samples 4900492 train steps 152828
Length 476
Episode 113255 total samples 4903165 train steps 152911
Length 505
Episode 113273 total samples 4905689 train steps 152990
Length 119
Episode 113284 total samples 4908250 train steps 153070
Length 660
Episode 113304 total samples 4910943 train steps 153154
Length 766
Episode 113322 total samples 4913463 train steps 153233
Length 103
exploration 0.0999991000096
Episode 113348 total samples 4916257 train steps 153321
Length 611
Episode 113379 total samples 4918910 train steps 153403
Length 390
Episode 113404 total samples 4921510 train steps 153485
Length 309
Episode 113424 total samples 4924059 train steps 153564
Length 299
Episode 113450 total samples 4926607 train steps 153644
Length 152
Episode 113478 total samples 4929450 train steps 153733
Length 696
Episode 113499 total samples 4932012 train steps 153813
Length 140
Episode 113513 total samples 4934564 train steps 153893
Length 1096
Episode 113530 total samples 4937071 train steps 153971
Length 168
Episode 113547 total samples 4939625 train steps 154051
Length 341
exploration 0.0999991000096
Episode 113559 total samples 4942128 train steps 154129
Length 104
Episode 113582 total samples 4944900 train steps 154216
Length 1231
Episode 113602 total samples 4947460 train steps 154296
Length 303
Episode 113619 total samples 4949969 train steps 154374
Length 157
Episode 113635 total samples 4952669 train steps 154458
Length 484
Episode 113663 total samples 4955183 train steps 154537
Length 144
Episode 113683 total samples 4957876 train steps 154621
Length 501
Episode 113697 total samples 4960444 train steps 154701
Length 346
Episode 113723 total samples 4963019 train steps 154782
Length 157
Episode 113741 total samples 4965597 train steps 154862
Length 423
exploration 0.0999991000096
Episode 113756 total samples 4968128 train steps 154941
Length 254
Episode 113777 total samples 4970639 train steps 155020
Length 159
Episode 113795 total samples 4973178 train steps 155099
Length 568
Episode 113817 total samples 4975876 train steps 155184
Length 421
Episode 113839 total samples 4978525 train steps 155266
Length 674
Episode 113857 total samples 4981057 train steps 155346
Length 146
Episode 113877 total samples 4983645 train steps 155426
Length 643
Episode 113890 total samples 4986413 train steps 155513
Length 1967
Episode 113910 total samples 4988926 train steps 155591
Length 54
exploration 0.0999991000096
Episode 113931 total samples 4991574 train steps 155674
Length 443
Episode 113957 total samples 4994080 train steps 155752
Length 121
Episode 113986 total samples 4996659 train steps 155833
Length 1052
Episode 114014 total samples 4999198 train steps 155912
Length 189
Episode 114034 total samples 5001754 train steps 155992
Length 405
Episode 114058 total samples 5004279 train steps 156071
Length 104
Episode 114078 total samples 5006922 train steps 156154
Length 291
Episode 114097 total samples 5009576 train steps 156237
Length 328
Episode 114120 total samples 5012162 train steps 156318
Length 192
Episode 114137 total samples 5014775 train steps 156399
Length 312
exploration 0.0999991000096
Episode 114155 total samples 5017293 train steps 156478
Length 338
Episode 114181 total samples 5020001 train steps 156563
Length 534
Episode 114201 total samples 5022587 train steps 156643
Length 685
Episode 114226 total samples 5025312 train steps 156728
Length 610
Episode 114248 total samples 5027944 train steps 156811
Length 478
Episode 114272 total samples 5030785 train steps 156900
Length 888
Episode 114289 total samples 5033359 train steps 156980
Length 149
Episode 114311 total samples 5036074 train steps 157065
Length 718
Episode 114328 total samples 5038719 train steps 157147
Length 348
Episode 114343 total samples 5041426 train steps 157232
Length 511
exploration 0.0999991000096
Episode 114358 total samples 5044093 train steps 157315
Length 369
Episode 114378 total samples 5047005 train steps 157406
Length 905
Episode 114400 total samples 5049808 train steps 157494
Length 723
Episode 114415 total samples 5052544 train steps 157579
Length 594
Episode 114437 total samples 5055188 train steps 157662
Length 815
Episode 114454 total samples 5057759 train steps 157742
Length 254
Episode 114497 total samples 5060375 train steps 157824
Length 246
Episode 114514 total samples 5063118 train steps 157910
Length 682
Episode 114528 total samples 5065838 train steps 157995
Length 955
exploration 0.0999991000096
Episode 114548 total samples 5068580 train steps 158081
Length 591
Episode 114563 total samples 5071180 train steps 158162
Length 248
Episode 114590 total samples 5073753 train steps 158242
Length 265
Episode 114616 total samples 5076269 train steps 158321
Length 73
Episode 114626 total samples 5078811 train steps 158400
Length 1843
Episode 114643 total samples 5081334 train steps 158479
Length 301
Episode 114663 total samples 5083942 train steps 158561
Length 413
Episode 114681 total samples 5086458 train steps 158639
Length 31
Episode 114698 total samples 5089078 train steps 158721
Length 563
exploration 0.0999991000096
Episode 114717 total samples 5091699 train steps 158803
Length 259
Episode 114733 total samples 5094297 train steps 158884
Length 301
Episode 114758 total samples 5096888 train steps 158965
Length 490
Episode 114774 total samples 5099400 train steps 159044
Length 86
Episode 114794 total samples 5101979 train steps 159124
Length 497
Episode 114821 total samples 5104561 train steps 159205
Length 476
Episode 114840 total samples 5107362 train steps 159293
Length 798
Episode 114874 total samples 5109891 train steps 159372
Length 443
Episode 114964 total samples 5112433 train steps 159451
Length 97
Episode 114997 total samples 5114950 train steps 159530
Length 271
exploration 0.0999991000096
Episode 115023 total samples 5117726 train steps 159616
Length 736
Episode 115044 total samples 5120307 train steps 159697
Length 371
Episode 115062 total samples 5122889 train steps 159778
Length 411
Episode 115080 total samples 5125417 train steps 159857
Length 112
Episode 115094 total samples 5127996 train steps 159937
Length 558
Episode 115117 total samples 5130617 train steps 160019
Length 547
Episode 115129 total samples 5133122 train steps 160098
Length 1309
Episode 115155 total samples 5135638 train steps 160176
Length 215
Episode 115177 total samples 5138186 train steps 160256
Length 160
Episode 115194 total samples 5140853 train steps 160339
Length 353
exploration 0.0999991000096
Episode 115224 total samples 5143373 train steps 160418
Length 170
Episode 115244 total samples 5145946 train steps 160498
Length 154
Episode 115261 total samples 5148485 train steps 160578
Length 405
Episode 115281 total samples 5151256 train steps 160664
Length 1073
Episode 115297 total samples 5153991 train steps 160750
Length 560
Episode 115327 total samples 5156732 train steps 160835
Length 493
Episode 115344 total samples 5159567 train steps 160924
Length 898
Episode 115362 total samples 5162150 train steps 161005
Length 324
Episode 115402 total samples 5164748 train steps 161086
Length 337
exploration 0.0999991000096
Episode 115420 total samples 5167311 train steps 161166
Length 474
Episode 115446 total samples 5169914 train steps 161247
Length 290
Episode 115467 total samples 5172514 train steps 161329
Length 266
Episode 115501 total samples 5175150 train steps 161411
Length 729
Episode 115523 total samples 5177807 train steps 161494
Length 485
Episode 115547 total samples 5180308 train steps 161572
Length 118
Episode 115572 total samples 5182916 train steps 161654
Length 338
Episode 115601 total samples 5185482 train steps 161734
Length 792
Episode 115636 total samples 5188032 train steps 161813
Length 170
Episode 115671 total samples 5190721 train steps 161898
Length 387
exploration 0.0999991000096
Episode 115691 total samples 5193320 train steps 161979
Length 309
Episode 115710 total samples 5195841 train steps 162058
Length 190
Episode 115736 total samples 5198402 train steps 162138
Length 220
Episode 115750 total samples 5200938 train steps 162217
Length 204
Episode 115770 total samples 5203449 train steps 162295
Length 110
Episode 115786 total samples 5206006 train steps 162375
Length 563
Episode 115811 total samples 5208621 train steps 162457
Length 332
Episode 115830 total samples 5211401 train steps 162544
Length 584
Episode 115851 total samples 5214378 train steps 162637
Length 1290
exploration 0.0999991000096
Episode 115874 total samples 5216931 train steps 162717
Length 109
Episode 115897 total samples 5219437 train steps 162795
Length 534
Episode 115917 total samples 5222114 train steps 162879
Length 409
Episode 115934 total samples 5224655 train steps 162958
Length 1092
Episode 115960 total samples 5227174 train steps 163037
Length 192
Episode 115988 total samples 5229737 train steps 163117
Length 176
Episode 116014 total samples 5232287 train steps 163196
Length 338
Episode 116038 total samples 5234796 train steps 163275
Length 370
Episode 116063 total samples 5237373 train steps 163355
Length 374
Episode 116089 total samples 5240032 train steps 163438
Length 398
exploration 0.0999991000096
Episode 116113 total samples 5242654 train steps 163520
Length 617
Episode 116144 total samples 5245293 train steps 163603
Length 350
Episode 116184 total samples 5247872 train steps 163683
Length 218
Episode 116210 total samples 5250438 train steps 163764
Length 181
Episode 116237 total samples 5253095 train steps 163847
Length 343
Episode 116265 total samples 5255611 train steps 163925
Length 39
Episode 116308 total samples 5258172 train steps 164005
Length 219
Episode 116349 total samples 5260877 train steps 164090
Length 458
Episode 116442 total samples 5263424 train steps 164169
Length 235
Episode 116465 total samples 5266069 train steps 164252
Length 1075
exploration 0.0999991000096
Episode 116492 total samples 5268585 train steps 164331
Length 489
Episode 116513 total samples 5271186 train steps 164412
Length 252
Episode 116539 total samples 5273738 train steps 164492
Length 240
Episode 116561 total samples 5276260 train steps 164571
Length 371
Episode 116582 total samples 5278852 train steps 164652
Length 310
Episode 116597 total samples 5281399 train steps 164731
Length 194
Episode 116620 total samples 5284104 train steps 164816
Length 411
Episode 116639 total samples 5286748 train steps 164898
Length 691
Episode 116663 total samples 5289263 train steps 164977
Length 640
Episode 116696 total samples 5291766 train steps 165055
Length 27
exploration 0.0999991000096
Episode 116725 total samples 5294338 train steps 165136
Length 294
Episode 116749 total samples 5296869 train steps 165215
Length 168
Episode 116779 total samples 5299469 train steps 165296
Length 458
Episode 116805 total samples 5302396 train steps 165387
Length 871
Episode 116820 total samples 5305081 train steps 165471
Length 749
Episode 116835 total samples 5307710 train steps 165553
Length 399
Episode 116861 total samples 5310232 train steps 165632
Length 165
Episode 116890 total samples 5312871 train steps 165715
Length 351
Episode 116907 total samples 5315434 train steps 165795
Length 136
exploration 0.0999991000096
Episode 116926 total samples 5318067 train steps 165877
Length 326
Episode 116948 total samples 5320611 train steps 165957
Length 475
Episode 116963 total samples 5323159 train steps 166036
Length 306
Episode 116979 total samples 5325739 train steps 166117
Length 317
Episode 116995 total samples 5328306 train steps 166197
Length 331
Episode 117014 total samples 5330837 train steps 166276
Length 276
Episode 117029 total samples 5333395 train steps 166356
Length 236
Episode 117046 total samples 5335934 train steps 166435
Length 284
Episode 117060 total samples 5338761 train steps 166524
Length 759
Episode 117091 total samples 5341480 train steps 166609
Length 570
exploration 0.0999991000096
Episode 117112 total samples 5344115 train steps 166691
Length 282
Episode 117133 total samples 5346770 train steps 166774
Length 471
Episode 117156 total samples 5349453 train steps 166858
Length 429
Episode 117182 total samples 5352293 train steps 166947
Length 815
Episode 117202 total samples 5354818 train steps 167026
Length 372
Episode 117220 total samples 5357448 train steps 167108
Length 606
Episode 117236 total samples 5360111 train steps 167191
Length 1111
Episode 117249 total samples 5362792 train steps 167275
Length 925
Episode 117269 total samples 5365307 train steps 167353
Length 40
exploration 0.0999991000096
Episode 117296 total samples 5367905 train steps 167435
Length 358
Episode 117325 total samples 5370581 train steps 167518
Length 438
Episode 117346 total samples 5373106 train steps 167597
Length 102
Episode 117359 total samples 5375825 train steps 167682
Length 961
Episode 117382 total samples 5378345 train steps 167761
Length 312
Episode 117396 total samples 5380861 train steps 167839
Length 405
Episode 117411 total samples 5383578 train steps 167924
Length 1415
Episode 117429 total samples 5386119 train steps 168004
Length 225
Episode 117446 total samples 5388675 train steps 168084
Length 391
Episode 117459 total samples 5391216 train steps 168163
Length 196
exploration 0.0999991000096
Episode 117477 total samples 5393756 train steps 168242
Length 295
Episode 117535 total samples 5396258 train steps 168321
Length 134
Episode 117560 total samples 5398873 train steps 168402
Length 252
Episode 117575 total samples 5402145 train steps 168505
Length 2034
Episode 117604 total samples 5404688 train steps 168584
Length 332
Episode 117615 total samples 5407222 train steps 168663
Length 274
Episode 117642 total samples 5409775 train steps 168743
Length 145
Episode 117667 total samples 5412304 train steps 168822
Length 177
Episode 117684 total samples 5414929 train steps 168904
Length 299
exploration 0.0999991000096
Episode 117695 total samples 5417443 train steps 168983
Length 72
Episode 117710 total samples 5420013 train steps 169063
Length 673
Episode 117738 total samples 5422750 train steps 169148
Length 995
Episode 117754 total samples 5425305 train steps 169228
Length 217
Episode 117769 total samples 5428039 train steps 169314
Length 803
Episode 117792 total samples 5430737 train steps 169398
Length 486
Episode 117806 total samples 5433458 train steps 169483
Length 463
Episode 117831 total samples 5436086 train steps 169565
Length 683
Episode 117855 total samples 5438656 train steps 169645
Length 501
Episode 117876 total samples 5441544 train steps 169736
Length 992
exploration 0.0999991000096
Episode 117899 total samples 5444130 train steps 169817
Length 346
Episode 117920 total samples 5446846 train steps 169901
Length 1003
Episode 117939 total samples 5449542 train steps 169986
Length 597
Episode 117959 total samples 5452075 train steps 170065
Length 69
Episode 117981 total samples 5454712 train steps 170147
Length 530
Episode 117997 total samples 5458208 train steps 170256
Length 2639
Episode 118011 total samples 5460834 train steps 170339
Length 849
Episode 118039 total samples 5463650 train steps 170427
Length 1087
Episode 118049 total samples 5466157 train steps 170505
Length 57
exploration 0.0999991000096
Episode 118069 total samples 5468663 train steps 170583
Length 267
Episode 118086 total samples 5471212 train steps 170663
Length 159
Episode 118102 total samples 5473822 train steps 170744
Length 275
Episode 118116 total samples 5476459 train steps 170827
Length 975
Episode 118140 total samples 5479005 train steps 170906
Length 156
Episode 118165 total samples 5481749 train steps 170992
Length 743
Episode 118179 total samples 5484304 train steps 171072
Length 194
Episode 118198 total samples 5486938 train steps 171154
Length 354
Episode 118212 total samples 5489564 train steps 171236
Length 314
Episode 118238 total samples 5492141 train steps 171317
Length 202
exploration 0.0999991000096
Episode 118253 total samples 5494662 train steps 171396
Length 186
Episode 118273 total samples 5497195 train steps 171475
Length 188
Episode 118290 total samples 5499947 train steps 171561
Length 1022
Episode 118314 total samples 5502506 train steps 171641
Length 337
Episode 118331 total samples 5505102 train steps 171722
Length 196
Episode 118350 total samples 5508228 train steps 171820
Length 1312
Episode 118367 total samples 5510947 train steps 171905
Length 534
Episode 118385 total samples 5513714 train steps 171991
Length 553
Episode 118397 total samples 5516320 train steps 172072
Length 243
exploration 0.0999991000096
Episode 118412 total samples 5518911 train steps 172153
Length 767
Episode 118433 total samples 5521799 train steps 172244
Length 900
Episode 118458 total samples 5524311 train steps 172322
Length 73
Episode 118477 total samples 5527076 train steps 172409
Length 1373
Episode 118493 total samples 5529644 train steps 172489
Length 826
Episode 118509 total samples 5532154 train steps 172567
Length 82
Episode 118527 total samples 5534817 train steps 172651
Length 574
Episode 118537 total samples 5537377 train steps 172731
Length 1067
Episode 118555 total samples 5540196 train steps 172819
Length 981
exploration 0.0999991000096
Episode 118577 total samples 5542705 train steps 172897
Length 110
Episode 118599 total samples 5545291 train steps 172978
Length 358
Episode 118619 total samples 5547836 train steps 173057
Length 93
Episode 118635 total samples 5550570 train steps 173143
Length 578
Episode 118657 total samples 5553235 train steps 173226
Length 431
Episode 118682 total samples 5555832 train steps 173307
Length 543
Episode 118708 total samples 5558451 train steps 173389
Length 525
Episode 118731 total samples 5560992 train steps 173468
Length 132
Episode 118745 total samples 5563729 train steps 173554
Length 913
Episode 118764 total samples 5566521 train steps 173641
Length 851
exploration 0.0999991000096
Episode 118781 total samples 5569149 train steps 173723
Length 461
Episode 118799 total samples 5571669 train steps 173802
Length 352
Episode 118811 total samples 5574264 train steps 173883
Length 367
Episode 118834 total samples 5576951 train steps 173967
Length 659
Episode 118850 total samples 5579608 train steps 174050
Length 359
Episode 118884 total samples 5582215 train steps 174132
Length 458
Episode 118904 total samples 5584927 train steps 174216
Length 455
Episode 118925 total samples 5587483 train steps 174296
Length 202
Episode 118946 total samples 5590048 train steps 174376
Length 552
Episode 118970 total samples 5592761 train steps 174461
Length 570
exploration 0.0999991000096
Episode 118987 total samples 5595280 train steps 174540
Length 441
Episode 119003 total samples 5597836 train steps 174620
Length 367
Episode 119019 total samples 5600367 train steps 174699
Length 201
Episode 119045 total samples 5602890 train steps 174778
Length 51
Episode 119062 total samples 5605428 train steps 174857
Length 292
Episode 119080 total samples 5607987 train steps 174937
Length 126
Episode 119099 total samples 5610746 train steps 175023
Length 714
Episode 119117 total samples 5613314 train steps 175104
Length 421
Episode 119142 total samples 5616072 train steps 175190
Length 561
exploration 0.0999991000096
Episode 119167 total samples 5618687 train steps 175271
Length 334
Episode 119189 total samples 5621530 train steps 175360
Length 707
Episode 119240 total samples 5624054 train steps 175439
Length 176
Episode 119264 total samples 5626682 train steps 175521
Length 409
Episode 119280 total samples 5629205 train steps 175600
Length 295
Episode 119304 total samples 5631854 train steps 175683
Length 340
Episode 119323 total samples 5634431 train steps 175763
Length 676
Episode 119336 total samples 5636968 train steps 175843
Length 740
Episode 119352 total samples 5639470 train steps 175921
Length 30
Episode 119369 total samples 5642000 train steps 176000
Length 714
exploration 0.0999991000096
Episode 119387 total samples 5644537 train steps 176079
Length 797
Episode 119405 total samples 5647409 train steps 176169
Length 820
Episode 119437 total samples 5649958 train steps 176249
Length 398
Episode 119454 total samples 5652532 train steps 176329
Length 247
Episode 119473 total samples 5655081 train steps 176409
Length 522
Episode 119491 total samples 5657704 train steps 176491
Length 259
Episode 119509 total samples 5660212 train steps 176569
Length 114
Episode 119532 total samples 5662799 train steps 176650
Length 582
Episode 119551 total samples 5665337 train steps 176729
Length 166
Episode 119567 total samples 5668029 train steps 176813
Length 396
exploration 0.0999991000096
Episode 119587 total samples 5670606 train steps 176894
Length 550
Episode 119611 total samples 5673265 train steps 176977
Length 456
Episode 119636 total samples 5675820 train steps 177057
Length 198
Episode 119704 total samples 5678357 train steps 177136
Length 230
Episode 119726 total samples 5680943 train steps 177217
Length 297
Episode 119748 total samples 5683460 train steps 177296
Length 368
Episode 119764 total samples 5686453 train steps 177389
Length 1321
Episode 119776 total samples 5689041 train steps 177470
Length 452
Episode 119791 total samples 5691693 train steps 177553
Length 859
exploration 0.0999991000096
Episode 119810 total samples 5694435 train steps 177639
Length 549
Episode 119828 total samples 5696997 train steps 177719
Length 165
Episode 119845 total samples 5699578 train steps 177799
Length 162
Episode 119868 total samples 5702088 train steps 177878
Length 56
Episode 119898 total samples 5704637 train steps 177957
Length 156
Episode 119918 total samples 5707193 train steps 178037
Length 184
Episode 119950 total samples 5709820 train steps 178119
Length 302
Episode 119974 total samples 5712598 train steps 178206
Length 603
Episode 119996 total samples 5715124 train steps 178285
Length 397
Episode 120016 total samples 5717679 train steps 178365
Length 359
exploration 0.0999991000096
Episode 120038 total samples 5720248 train steps 178445
Length 193
Episode 120057 total samples 5722768 train steps 178524
Length 163
Episode 120077 total samples 5725317 train steps 178604
Length 295
Episode 120095 total samples 5727938 train steps 178686
Length 657
Episode 120119 total samples 5730439 train steps 178764
Length 65
Episode 120136 total samples 5733557 train steps 178861
Length 1358
Episode 120157 total samples 5736062 train steps 178939
Length 441
Episode 120176 total samples 5738688 train steps 179021
Length 306
Episode 120195 total samples 5741250 train steps 179102
Length 351
exploration 0.0999991000096
Episode 120211 total samples 5744007 train steps 179188
Length 517
Episode 120240 total samples 5746528 train steps 179266
Length 471
Episode 120266 total samples 5749059 train steps 179346
Length 311
Episode 120289 total samples 5751726 train steps 179429
Length 578
Episode 120311 total samples 5754361 train steps 179511
Length 406
Episode 120332 total samples 5756867 train steps 179590
Length 36
Episode 120350 total samples 5759396 train steps 179669
Length 226
Episode 120379 total samples 5761976 train steps 179749
Length 411
Episode 120389 total samples 5764541 train steps 179829
Length 146
Episode 120410 total samples 5767127 train steps 179910
Length 337
exploration 0.0999991000096
Episode 120429 total samples 5769637 train steps 179989
Length 530
Episode 120460 total samples 5772193 train steps 180069
Length 214
Episode 120480 total samples 5774862 train steps 180152
Length 367
Episode 120500 total samples 5777567 train steps 180236
Length 535
Episode 120523 total samples 5780166 train steps 180318
Length 214
Episode 120549 total samples 5782696 train steps 180397
Length 186
Episode 120568 total samples 5785546 train steps 180486
Length 828
Episode 120584 total samples 5788232 train steps 180570
Length 782
Episode 120607 total samples 5790775 train steps 180649
Length 117
Episode 120638 total samples 5793337 train steps 180729
Length 935
exploration 0.0999991000096
In [ ]:
car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
In [1]:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
In [2]:
from dqn_learning import *
# import cPickle
# %load_ext autoreload
# %autoreload 2
%pylab inline
# import numpy as np
[2016-08-25 12:12:31,925] Site environment registry incorrect: Scoreboard did not register all envs: set(['AcrobotContinuous-v0'])
Populating the interactive namespace from numpy and matplotlib
WARNING: pylab import has clobbered these variables: ['copy']
`%matplotlib` prevents importing * from pylab and numpy
In [3]:
car = q_learning(gamma=0.99,
# lambda_=0.9,
# init_epsilon = 0.0,
# update_epsilon = False,
init_epsilon = 1.0,
end_epsilon = 0.1,
update_epsilon = True,
exploration_decrease_length = 1e6,
# N_0= 50,
environment = 'CartPole-v0',
plot_resolution = 30,
nn_size_hidden = [400,300], # [100,100] --> 0.01, [100,100,100] --> similar [400,300]
nn_batch_size = 128,
nn_learning_rate = 2.5e-4,
replay_memory_size = 3e5, # larger than nn_batch_size*1e3 or doesn't train
descent_method = 'adam',
# dropout_keep_prob = 0.9,
ema_decay_rate = 0.999,
train_frequency = 4.0,
from_pixels = True,
reg_weight = 0.0,
do_pretrain = False,
pretrain_steps = 0
)
[2016-08-25 12:12:34,135] Making new env: CartPole-v0
conv 128 8 8 2
weights std = 0.125
[8, 8, 2, 32]
bias std = 0.0
conv 512 4 4 32
weights std = 0.0625
[4, 4, 32, 64]
bias std = 0.0
conv 576 3 3 64
weights std = 0.0589255650989
[3, 3, 64, 64]
bias std = 0.0
fc 7744
weights std = 0.0160706086633
bias std = 0.0
fc 512
weights std = 0.0441941738242
bias std = 0.0
Q-learning configured in nn graph
descending with adam
using environment CartPole-v0
qnn target q-learning False False
In [4]:
car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
Episode 1036 total samples 10048 train steps 1
Length 65
Episode 1295 total samples 12549 train steps 80
Length 14
Episode 1542 total samples 15053 train steps 158
Length 10
Episode 1822 total samples 17556 train steps 236
Length 12
Episode 2102 total samples 20064 train steps 314
Length 19
Episode 2352 total samples 22568 train steps 393
Length 9
exploration 0.977497300001
/usr/lib/pymodules/python2.7/matplotlib/collections.py:548: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
if self._edgecolors == 'face':
Episode 2617 total samples 25070 train steps 471
Length 34
Episode 2862 total samples 27575 train steps 549
Length 26
Episode 3127 total samples 30090 train steps 628
Length 33
Episode 3385 total samples 32591 train steps 706
Length 8
Episode 3640 total samples 35092 train steps 784
Length 9
Episode 3898 total samples 37599 train steps 862
Length 14
Episode 4162 total samples 40103 train steps 941
Length 9
Episode 4426 total samples 42607 train steps 1019
Length 16
Episode 4692 total samples 45115 train steps 1097
Length 25
Episode 4955 total samples 47616 train steps 1175
Length 10
exploration 0.954993700001
Episode 5212 total samples 50118 train steps 1254
Length 16
Episode 5478 total samples 52623 train steps 1332
Length 10
Episode 5752 total samples 55127 train steps 1410
Length 14
Episode 6022 total samples 57628 train steps 1488
Length 12
Episode 6285 total samples 60133 train steps 1567
Length 21
Episode 6558 total samples 62641 train steps 1645
Length 23
Episode 6828 total samples 65142 train steps 1723
Length 23
Episode 7075 total samples 67657 train steps 1802
Length 45
Episode 7334 total samples 70159 train steps 1880
Length 10
Episode 7591 total samples 72660 train steps 1958
Length 9
exploration 0.932490100002
Episode 7846 total samples 75164 train steps 2036
Length 19
Episode 8117 total samples 77687 train steps 2115
Length 50
Episode 8375 total samples 80190 train steps 2193
Length 14
Episode 8635 total samples 82697 train steps 2272
Length 20
Episode 8892 total samples 85214 train steps 2350
Length 34
Episode 9160 total samples 87726 train steps 2429
Length 24
Episode 9429 total samples 90234 train steps 2507
Length 19
Episode 9666 total samples 92743 train steps 2586
Length 54
Episode 9919 total samples 95246 train steps 2664
Length 12
Episode 10162 total samples 97759 train steps 2742
Length 25
exploration 0.909984700003
Episode 10431 total samples 100264 train steps 2821
Length 15
Episode 10689 total samples 102769 train steps 2899
Length 17
Episode 10941 total samples 105275 train steps 2977
Length 12
Episode 11213 total samples 107777 train steps 3056
Length 26
Episode 11462 total samples 110281 train steps 3134
Length 10
Episode 11720 total samples 112783 train steps 3212
Length 9
Episode 11978 total samples 115286 train steps 3290
Length 14
Episode 12230 total samples 117797 train steps 3369
Length 41
Episode 12492 total samples 120298 train steps 3447
Length 17
Episode 12731 total samples 122800 train steps 3525
Length 24
exploration 0.887482000004
Episode 12971 total samples 125315 train steps 3604
Length 57
Episode 13212 total samples 127821 train steps 3682
Length 12
Episode 13454 total samples 130328 train steps 3760
Length 19
Episode 13694 total samples 132853 train steps 3839
Length 54
Episode 13950 total samples 135358 train steps 3917
Length 11
Episode 14197 total samples 137863 train steps 3996
Length 14
Episode 14437 total samples 140367 train steps 4074
Length 10
Episode 14672 total samples 142869 train steps 4152
Length 16
Episode 14916 total samples 145376 train steps 4230
Length 16
Episode 15155 total samples 147885 train steps 4309
Length 31
exploration 0.864980200004
Episode 15403 total samples 150393 train steps 4387
Length 49
Episode 15632 total samples 152905 train steps 4466
Length 56
Episode 15874 total samples 155407 train steps 4544
Length 34
Episode 16100 total samples 157916 train steps 4622
Length 21
Episode 16344 total samples 160428 train steps 4701
Length 35
Episode 16598 total samples 162938 train steps 4779
Length 27
Episode 16838 total samples 165444 train steps 4858
Length 21
Episode 17072 total samples 167945 train steps 4936
Length 10
Episode 17295 total samples 170450 train steps 5014
Length 28
Episode 17546 total samples 172951 train steps 5092
Length 27
exploration 0.842471200005
Episode 17777 total samples 175457 train steps 5171
Length 13
Episode 17999 total samples 177963 train steps 5249
Length 14
Episode 18217 total samples 180475 train steps 5327
Length 23
Episode 18437 total samples 182976 train steps 5405
Length 42
Episode 18663 total samples 185486 train steps 5484
Length 21
Episode 18897 total samples 187988 train steps 5562
Length 15
Episode 19115 total samples 190492 train steps 5640
Length 25
Episode 19353 total samples 192998 train steps 5719
Length 39
Episode 19580 total samples 195502 train steps 5797
Length 18
Episode 19803 total samples 198004 train steps 5875
Length 12
exploration 0.819964900006
Episode 20028 total samples 200510 train steps 5953
Length 20
Episode 20240 total samples 203017 train steps 6032
Length 35
Episode 20461 total samples 205518 train steps 6110
Length 36
Episode 20682 total samples 208024 train steps 6188
Length 14
Episode 20902 total samples 210535 train steps 6267
Length 28
Episode 21127 total samples 213038 train steps 6345
Length 30
Episode 21339 total samples 215540 train steps 6423
Length 22
Episode 21538 total samples 218050 train steps 6502
Length 53
Episode 21760 total samples 220555 train steps 6580
Length 10
Episode 21971 total samples 223058 train steps 6658
Length 13
exploration 0.797440600007
Episode 22182 total samples 225560 train steps 6736
Length 13
Episode 22377 total samples 228066 train steps 6815
Length 58
Episode 22578 total samples 230568 train steps 6893
Length 14
Episode 22783 total samples 233071 train steps 6971
Length 30
Episode 22980 total samples 235577 train steps 7049
Length 13
Episode 23176 total samples 238085 train steps 7128
Length 17
Episode 23374 total samples 240590 train steps 7206
Length 13
Episode 23579 total samples 243091 train steps 7284
Length 27
Episode 23787 total samples 245600 train steps 7362
Length 21
Episode 24000 total samples 248107 train steps 7441
Length 16
exploration 0.774926200007
Episode 24187 total samples 250609 train steps 7519
Length 29
Episode 24399 total samples 253134 train steps 7598
Length 50
Episode 24606 total samples 255644 train steps 7676
Length 52
Episode 24801 total samples 258150 train steps 7755
Length 24
Episode 25002 total samples 260658 train steps 7833
Length 24
Episode 25207 total samples 263159 train steps 7911
Length 27
Episode 25398 total samples 265664 train steps 7989
Length 31
Episode 25592 total samples 268176 train steps 8068
Length 35
Episode 25778 total samples 270688 train steps 8146
Length 27
Episode 25986 total samples 273195 train steps 8225
Length 27
exploration 0.752418100008
Episode 26187 total samples 275700 train steps 8303
Length 40
Episode 26375 total samples 278205 train steps 8381
Length 38
Episode 26585 total samples 280706 train steps 8460
Length 13
Episode 26777 total samples 283210 train steps 8538
Length 65
Episode 26980 total samples 285719 train steps 8616
Length 17
Episode 27168 total samples 288221 train steps 8694
Length 34
Episode 27370 total samples 290722 train steps 8773
Length 28
Episode 27556 total samples 293225 train steps 8851
Length 10
Episode 27739 total samples 295727 train steps 8929
Length 13
Episode 27934 total samples 298243 train steps 9008
Length 31
exploration 0.729898300009
Episode 28121 total samples 300747 train steps 9086
Length 38
Episode 28307 total samples 303258 train steps 9164
Length 26
Episode 28501 total samples 305768 train steps 9243
Length 26
Episode 28674 total samples 308282 train steps 9321
Length 37
Episode 28859 total samples 310804 train steps 9400
Length 71
Episode 29038 total samples 313323 train steps 9479
Length 73
Episode 29212 total samples 315844 train steps 9558
Length 46
Episode 29411 total samples 318354 train steps 9636
Length 33
Episode 29578 total samples 320862 train steps 9714
Length 15
Episode 29752 total samples 323383 train steps 9793
Length 66
exploration 0.70738300001
Episode 29921 total samples 325895 train steps 9872
Length 31
Episode 30096 total samples 328400 train steps 9950
Length 14
Episode 30267 total samples 330912 train steps 10028
Length 24
Episode 30442 total samples 333423 train steps 10107
Length 35
Episode 30612 total samples 335956 train steps 10186
Length 89
Episode 30779 total samples 338463 train steps 10264
Length 21
Episode 30943 total samples 340971 train steps 10343
Length 26
Episode 31114 total samples 343477 train steps 10421
Length 30
Episode 31274 total samples 345979 train steps 10499
Length 60
Episode 31430 total samples 348484 train steps 10578
Length 46
exploration 0.68487760001
Episode 31615 total samples 350997 train steps 10656
Length 55
Episode 31788 total samples 353509 train steps 10735
Length 90
Episode 31955 total samples 356018 train steps 10813
Length 43
Episode 32117 total samples 358519 train steps 10891
Length 13
Episode 32283 total samples 361031 train steps 10970
Length 42
Episode 32448 total samples 363536 train steps 11048
Length 22
Episode 32612 total samples 366038 train steps 11126
Length 17
Episode 32781 total samples 368551 train steps 11205
Length 37
Episode 32943 total samples 371062 train steps 11283
Length 34
Episode 33102 total samples 373570 train steps 11362
Length 42
exploration 0.662370400011
Episode 33257 total samples 376078 train steps 11440
Length 15
Episode 33417 total samples 378585 train steps 11518
Length 67
Episode 33568 total samples 381095 train steps 11597
Length 35
Episode 33737 total samples 383598 train steps 11675
Length 13
Episode 33897 total samples 386102 train steps 11753
Length 12
Episode 34058 total samples 388615 train steps 11832
Length 52
Episode 34215 total samples 391138 train steps 11911
Length 67
Episode 34364 total samples 393653 train steps 11989
Length 33
Episode 34524 total samples 396162 train steps 12068
Length 22
Episode 34674 total samples 398705 train steps 12147
Length 106
exploration 0.639850600012
Episode 34825 total samples 401223 train steps 12226
Length 49
Episode 34982 total samples 403728 train steps 12304
Length 162
Episode 35138 total samples 406258 train steps 12383
Length 68
Episode 35285 total samples 408786 train steps 12462
Length 64
Episode 35435 total samples 411289 train steps 12540
Length 14
Episode 35598 total samples 413796 train steps 12619
Length 30
Episode 35769 total samples 416314 train steps 12697
Length 41
Episode 35927 total samples 418835 train steps 12776
Length 75
Episode 36063 total samples 421340 train steps 12854
Length 33
Episode 36222 total samples 423844 train steps 12933
Length 39
exploration 0.617343400013
Episode 36376 total samples 426354 train steps 13011
Length 29
Episode 36518 total samples 428863 train steps 13089
Length 32
Episode 36654 total samples 431368 train steps 13168
Length 19
Episode 36805 total samples 433881 train steps 13246
Length 47
Episode 36945 total samples 436390 train steps 13325
Length 17
Episode 37091 total samples 438896 train steps 13403
Length 14
Episode 37249 total samples 441422 train steps 13482
Length 71
Episode 37403 total samples 443932 train steps 13560
Length 31
Episode 37549 total samples 446449 train steps 13639
Length 75
Episode 37683 total samples 448953 train steps 13717
Length 20
exploration 0.594811000013
Episode 37829 total samples 451459 train steps 13796
Length 59
Episode 37966 total samples 453967 train steps 13874
Length 31
Episode 38090 total samples 456493 train steps 13953
Length 57
Episode 38241 total samples 459000 train steps 14031
Length 36
Episode 38396 total samples 461514 train steps 14110
Length 52
Episode 38531 total samples 464026 train steps 14188
Length 91
Episode 38680 total samples 466531 train steps 14267
Length 9
Episode 38826 total samples 469032 train steps 14345
Length 17
Episode 38966 total samples 471542 train steps 14423
Length 33
Episode 39105 total samples 474050 train steps 14502
Length 31
exploration 0.572309200014
Episode 39243 total samples 476555 train steps 14580
Length 27
Episode 39374 total samples 479104 train steps 14659
Length 101
Episode 39515 total samples 481610 train steps 14738
Length 36
Episode 39654 total samples 484148 train steps 14817
Length 84
Episode 39797 total samples 486650 train steps 14895
Length 22
Episode 39941 total samples 489174 train steps 14974
Length 64
Episode 40080 total samples 491683 train steps 15053
Length 33
Episode 40205 total samples 494194 train steps 15131
Length 21
Episode 40343 total samples 496701 train steps 15209
Length 18
Episode 40469 total samples 499237 train steps 15289
Length 106
exploration 0.549804700015
Episode 40605 total samples 501763 train steps 15368
Length 58
Episode 40721 total samples 504296 train steps 15447
Length 88
Episode 40854 total samples 506797 train steps 15525
Length 24
Episode 40968 total samples 509337 train steps 15604
Length 132
Episode 41094 total samples 511841 train steps 15683
Length 41
Episode 41228 total samples 514344 train steps 15761
Length 36
Episode 41358 total samples 516862 train steps 15839
Length 95
Episode 41484 total samples 519375 train steps 15918
Length 28
Episode 41590 total samples 521876 train steps 15996
Length 38
Episode 41731 total samples 524377 train steps 16074
Length 51
exploration 0.527299300016
Episode 41863 total samples 526878 train steps 16152
Length 47
Episode 41980 total samples 529411 train steps 16232
Length 99
Episode 42093 total samples 531929 train steps 16310
Length 36
Episode 42204 total samples 534462 train steps 16389
Length 81
Episode 42314 total samples 536966 train steps 16468
Length 48
Episode 42436 total samples 539471 train steps 16546
Length 63
Episode 42550 total samples 541993 train steps 16625
Length 96
[2016-08-25 13:29:19,874] Observation '[-2.4147833 -2.39643029 -0.14644597 0.26079187]' is not contained within observation space 'Box(4,)'.
Episode 42671 total samples 544495 train steps 16703
Length 23
Episode 42786 total samples 547020 train steps 16782
Length 69
Episode 42906 total samples 549542 train steps 16861
Length 51
exploration 0.504798400016
Episode 43023 total samples 552054 train steps 16939
Length 42
Episode 43127 total samples 554564 train steps 17018
Length 21
Episode 43245 total samples 557092 train steps 17097
Length 56
Episode 43360 total samples 559607 train steps 17175
Length 37
Episode 43479 total samples 562111 train steps 17253
Length 29
Episode 43593 total samples 564624 train steps 17332
Length 29
Episode 43716 total samples 567128 train steps 17410
Length 17
Episode 43837 total samples 569646 train steps 17489
Length 38
Episode 43954 total samples 572154 train steps 17567
Length 75
Episode 44067 total samples 574660 train steps 17646
Length 29
exploration 0.482294800016
Episode 44189 total samples 577173 train steps 17724
Length 53
Episode 44293 total samples 579693 train steps 17803
Length 61
Episode 44416 total samples 582224 train steps 17882
Length 87
Episode 44516 total samples 584734 train steps 17960
Length 47
Episode 44638 total samples 587285 train steps 18040
Length 108
Episode 44764 total samples 589811 train steps 18119
Length 99
Episode 44876 total samples 592327 train steps 18198
Length 90
Episode 44985 total samples 594830 train steps 18276
Length 10
Episode 45101 total samples 597344 train steps 18354
Length 103
Episode 45203 total samples 599846 train steps 18433
Length 17
exploration 0.459754300015
Episode 45311 total samples 602368 train steps 18511
Length 49
Episode 45404 total samples 604873 train steps 18590
Length 98
Episode 45520 total samples 607408 train steps 18669
Length 98
Episode 45636 total samples 609979 train steps 18749
Length 144
Episode 45740 total samples 612506 train steps 18828
Length 162
Episode 45826 total samples 615028 train steps 18907
Length 113
Episode 45934 total samples 617535 train steps 18985
Length 30
Episode 46029 total samples 620067 train steps 19065
Length 70
Episode 46126 total samples 622570 train steps 19143
Length 95
Episode 46222 total samples 625104 train steps 19222
Length 75
exploration 0.437223700015
Episode 46319 total samples 627650 train steps 19302
Length 105
Episode 46426 total samples 630152 train steps 19380
Length 24
Episode 46524 total samples 632668 train steps 19458
Length 39
Episode 46623 total samples 635181 train steps 19537
Length 41
Episode 46718 total samples 637697 train steps 19616
Length 72
Episode 46813 total samples 640210 train steps 19694
Length 42
Episode 46912 total samples 642722 train steps 19773
Length 24
Episode 46990 total samples 645223 train steps 19851
Length 33
Episode 47086 total samples 647789 train steps 19931
Length 141
Episode 47179 total samples 650293 train steps 20009
Length 22
exploration 0.414691300014
Episode 47272 total samples 652799 train steps 20087
Length 68
Episode 47367 total samples 655302 train steps 20166
Length 23
Episode 47458 total samples 657803 train steps 20244
Length 59
Episode 47543 total samples 660316 train steps 20322
Length 64
Episode 47642 total samples 662828 train steps 20401
Length 109
Episode 47727 total samples 665343 train steps 20479
Length 33
Episode 47822 total samples 667886 train steps 20559
Length 103
Episode 47917 total samples 670399 train steps 20637
Length 68
Episode 48016 total samples 672959 train steps 20717
Length 157
exploration 0.392178700013
Episode 48109 total samples 675467 train steps 20796
Length 15
Episode 48195 total samples 677982 train steps 20874
Length 34
Episode 48274 total samples 680513 train steps 20954
Length 72
Episode 48375 total samples 683032 train steps 21032
Length 40
Episode 48461 total samples 685606 train steps 21113
Length 147
Episode 48540 total samples 688165 train steps 21193
Length 119
Episode 48630 total samples 690671 train steps 21271
Length 142
Episode 48708 total samples 693223 train steps 21351
Length 108
Episode 48783 total samples 695729 train steps 21429
Length 18
Episode 48874 total samples 698280 train steps 21509
Length 108
exploration 0.369667000013
Episode 48954 total samples 700799 train steps 21587
Length 118
Episode 49039 total samples 703313 train steps 21666
Length 41
Episode 49121 total samples 705831 train steps 21745
Length 105
Episode 49201 total samples 708378 train steps 21824
Length 109
Episode 49283 total samples 710887 train steps 21903
Length 26
Episode 49362 total samples 713391 train steps 21981
Length 49
Episode 49439 total samples 715910 train steps 22060
Length 139
Episode 49519 total samples 718440 train steps 22139
Length 92
Episode 49592 total samples 720980 train steps 22218
Length 90
Episode 49672 total samples 723517 train steps 22297
Length 76
exploration 0.347157100012
Episode 49748 total samples 726064 train steps 22377
Length 106
Episode 49820 total samples 728576 train steps 22455
Length 116
Episode 49891 total samples 731084 train steps 22534
Length 45
Episode 49968 total samples 733627 train steps 22613
Length 116
Episode 50045 total samples 736134 train steps 22692
Length 43
Episode 50125 total samples 738677 train steps 22771
Length 109
Episode 50198 total samples 741197 train steps 22850
Length 155
Episode 50277 total samples 743747 train steps 22930
Length 113
Episode 50353 total samples 746256 train steps 23008
Length 43
Episode 50427 total samples 748789 train steps 23087
Length 66
exploration 0.324615700011
Episode 50489 total samples 751310 train steps 23166
Length 72
Episode 50572 total samples 753816 train steps 23244
Length 63
Episode 50659 total samples 756347 train steps 23323
Length 82
Episode 50718 total samples 758860 train steps 23402
Length 61
Episode 50780 total samples 761404 train steps 23481
Length 111
Episode 50848 total samples 763942 train steps 23561
Length 117
Episode 50913 total samples 766458 train steps 23639
Length 49
Episode 50989 total samples 768995 train steps 23719
Length 135
Episode 51053 total samples 771517 train steps 23797
Length 143
Episode 51124 total samples 774030 train steps 23876
Length 155
exploration 0.302050900011
Episode 51196 total samples 776539 train steps 23954
Length 50
Episode 51264 total samples 779063 train steps 24033
Length 135
Episode 51333 total samples 781620 train steps 24113
Length 121
Episode 51406 total samples 784176 train steps 24193
Length 177
Episode 51475 total samples 786686 train steps 24271
Length 72
Episode 51533 total samples 789189 train steps 24350
Length 123
Episode 51594 total samples 791710 train steps 24428
Length 149
Episode 51653 total samples 794274 train steps 24509
Length 151
Episode 51711 total samples 796785 train steps 24587
Length 118
Episode 51773 total samples 799291 train steps 24665
Length 21
exploration 0.27953110001
Episode 51839 total samples 801805 train steps 24744
Length 122
Episode 51895 total samples 804344 train steps 24823
Length 127
Episode 51954 total samples 806863 train steps 24902
Length 67
Episode 52013 total samples 809377 train steps 24981
Length 110
Episode 52066 total samples 811879 train steps 25059
Length 9
Episode 52127 total samples 814425 train steps 25138
Length 131
Episode 52184 total samples 816963 train steps 25218
Length 119
Episode 52250 total samples 819466 train steps 25296
Length 93
Episode 52307 total samples 821982 train steps 25374
Length 38
Episode 52367 total samples 824522 train steps 25454
Length 151
exploration 0.257019400009
Episode 52419 total samples 827029 train steps 25532
Length 97
Episode 52478 total samples 829553 train steps 25611
Length 124
Episode 52535 total samples 832060 train steps 25689
Length 142
Episode 52583 total samples 834617 train steps 25769
Length 114
Episode 52640 total samples 837145 train steps 25848
Length 132
Episode 52693 total samples 839680 train steps 25927
Length 115
Episode 52739 total samples 842242 train steps 26008
Length 215
Episode 52794 total samples 844784 train steps 26087
Length 109
Episode 52838 total samples 847323 train steps 26166
Length 187
Episode 52882 total samples 849835 train steps 26245
Length 128
exploration 0.234466300009
Episode 52932 total samples 852353 train steps 26324
Length 110
Episode 52987 total samples 854863 train steps 26402
Length 69
Episode 53035 total samples 857388 train steps 26481
Length 164
Episode 53090 total samples 859912 train steps 26560
Length 64
Episode 53142 total samples 862466 train steps 26640
Length 154
Episode 53196 total samples 864987 train steps 26718
Length 45
Episode 53242 total samples 867535 train steps 26798
Length 143
Episode 53289 total samples 870077 train steps 26877
Length 138
Episode 53336 total samples 872599 train steps 26956
Length 129
Episode 53381 total samples 875136 train steps 27035
Length 135
exploration 0.211906000009
Episode 53428 total samples 877714 train steps 27116
Length 194
Episode 53479 total samples 880229 train steps 27195
Length 127
Episode 53520 total samples 882801 train steps 27275
Length 152
Episode 53568 total samples 885356 train steps 27355
Length 183
Episode 53613 total samples 887902 train steps 27434
Length 167
Episode 53657 total samples 890467 train steps 27515
Length 159
Episode 53698 total samples 893004 train steps 27594
Length 111
Episode 53742 total samples 895570 train steps 27674
Length 213
Episode 53784 total samples 898130 train steps 27754
Length 238
Episode 53833 total samples 900656 train steps 27833
Length 92
exploration 0.189338500009
Episode 53873 total samples 903172 train steps 27912
Length 237
Episode 53916 total samples 905735 train steps 27992
Length 135
Episode 53957 total samples 908300 train steps 28072
Length 140
Episode 54001 total samples 910817 train steps 28151
Length 74
Episode 54046 total samples 913364 train steps 28230
Length 158
Episode 54084 total samples 915940 train steps 28311
Length 209
Episode 54124 total samples 918441 train steps 28389
Length 156
Episode 54161 total samples 920969 train steps 28468
Length 294
Episode 54201 total samples 923471 train steps 28546
Length 129
exploration 0.166834000009
Episode 54239 total samples 925991 train steps 28625
Length 180
Episode 54280 total samples 928626 train steps 28707
Length 272
Episode 54318 total samples 931144 train steps 28786
Length 109
Episode 54361 total samples 933665 train steps 28865
Length 169
Episode 54394 total samples 936265 train steps 28946
Length 207
Episode 54429 total samples 938858 train steps 29027
Length 226
Episode 54464 total samples 941440 train steps 29107
Length 164
Episode 54496 total samples 944004 train steps 29188
Length 182
Episode 54527 total samples 946556 train steps 29267
Length 287
Episode 54561 total samples 949098 train steps 29347
Length 138
exploration 0.144282700009
Episode 54595 total samples 951637 train steps 29426
Length 95
Episode 54630 total samples 954147 train steps 29505
Length 149
Episode 54663 total samples 956739 train steps 29586
Length 312
Episode 54696 total samples 959354 train steps 29667
Length 241
Episode 54725 total samples 961891 train steps 29747
Length 139
Episode 54754 total samples 964415 train steps 29825
Length 145
Episode 54779 total samples 966947 train steps 29905
Length 115
Episode 54804 total samples 969467 train steps 29983
Length 239
Episode 54831 total samples 971977 train steps 30062
Length 141
Episode 54862 total samples 974518 train steps 30141
Length 126
exploration 0.12169450001
Episode 54889 total samples 977087 train steps 30221
Length 413
Episode 54918 total samples 979673 train steps 30302
Length 275
Episode 54946 total samples 982268 train steps 30383
Length 326
Episode 54974 total samples 984826 train steps 30463
Length 146
Episode 54998 total samples 987391 train steps 30543
Length 139
Episode 55029 total samples 989990 train steps 30625
Length 324
Episode 55055 total samples 992663 train steps 30708
Length 490
Episode 55080 total samples 995276 train steps 30790
Length 502
Episode 55108 total samples 997843 train steps 30870
Length 238
Episode 55133 total samples 1000416 train steps 30950
Length 220
exploration 0.0999991000096
Episode 55156 total samples 1002937 train steps 31029
Length 386
Episode 55179 total samples 1005581 train steps 31112
Length 350
Episode 55200 total samples 1008338 train steps 31198
Length 815
Episode 55226 total samples 1011121 train steps 31285
Length 625
Episode 55246 total samples 1013678 train steps 31365
Length 322
Episode 55272 total samples 1016225 train steps 31445
Length 430
Episode 55297 total samples 1018982 train steps 31531
Length 535
Episode 55320 total samples 1021780 train steps 31618
Length 619
Episode 55341 total samples 1024422 train steps 31701
Length 309
exploration 0.0999991000096
Episode 55361 total samples 1027029 train steps 31782
Length 310
Episode 55379 total samples 1029593 train steps 31862
Length 396
Episode 55399 total samples 1032098 train steps 31941
Length 174
Episode 55422 total samples 1034855 train steps 32027
Length 718
Episode 55441 total samples 1037577 train steps 32112
Length 855
Episode 55460 total samples 1040094 train steps 32190
Length 619
Episode 55482 total samples 1042629 train steps 32270
Length 645
Episode 55503 total samples 1045141 train steps 32348
Length 265
Episode 55523 total samples 1047643 train steps 32426
Length 163
Episode 55551 total samples 1050230 train steps 32507
Length 345
exploration 0.0999991000096
Episode 55570 total samples 1052828 train steps 32588
Length 503
Episode 55590 total samples 1055550 train steps 32673
Length 581
Episode 55615 total samples 1058073 train steps 32752
Length 112
Episode 55637 total samples 1060674 train steps 32834
Length 214
Episode 55663 total samples 1063202 train steps 32913
Length 202
Episode 55684 total samples 1066299 train steps 33009
Length 1212
Episode 55710 total samples 1068946 train steps 33092
Length 326
Episode 55730 total samples 1071499 train steps 33172
Length 215
Episode 55747 total samples 1074234 train steps 33257
Length 475
exploration 0.0999991000096
Episode 55764 total samples 1076786 train steps 33337
Length 392
Episode 55783 total samples 1079317 train steps 33416
Length 269
Episode 55800 total samples 1081921 train steps 33498
Length 268
Episode 55815 total samples 1084590 train steps 33581
Length 707
Episode 55837 total samples 1087124 train steps 33660
Length 161
Episode 55863 total samples 1089820 train steps 33744
Length 576
Episode 55887 total samples 1092415 train steps 33825
Length 403
Episode 55910 total samples 1094942 train steps 33904
Length 133
Episode 55932 total samples 1097492 train steps 33984
Length 138
Episode 55958 total samples 1100070 train steps 34065
Length 203
exploration 0.0999991000096
Episode 55978 total samples 1102630 train steps 34145
Length 230
Episode 56006 total samples 1105140 train steps 34223
Length 197
Episode 56023 total samples 1107664 train steps 34302
Length 315
Episode 56049 total samples 1110465 train steps 34390
Length 667
Episode 56074 total samples 1112988 train steps 34468
Length 207
Episode 56099 total samples 1115504 train steps 34547
Length 40
Episode 56127 total samples 1118045 train steps 34626
Length 176
Episode 56146 total samples 1120579 train steps 34706
Length 247
Episode 56170 total samples 1123223 train steps 34788
Length 335
Episode 56192 total samples 1125745 train steps 34867
Length 216
exploration 0.0999991000096
Episode 56215 total samples 1128263 train steps 34946
Length 131
Episode 56232 total samples 1130920 train steps 35029
Length 564
Episode 56251 total samples 1133494 train steps 35109
Length 216
Episode 56268 total samples 1135999 train steps 35187
Length 410
Episode 56284 total samples 1138887 train steps 35278
Length 984
Episode 56305 total samples 1141399 train steps 35356
Length 250
Episode 56323 total samples 1143969 train steps 35437
Length 296
Episode 56343 total samples 1146660 train steps 35521
Length 474
Episode 56365 total samples 1149483 train steps 35609
Length 678
exploration 0.0999991000096
Episode 56388 total samples 1152030 train steps 35688
Length 199
Episode 56410 total samples 1154538 train steps 35767
Length 69
Episode 56423 total samples 1157153 train steps 35849
Length 246
Episode 56444 total samples 1159688 train steps 35928
Length 222
Episode 56461 total samples 1162343 train steps 36011
Length 610
Episode 56480 total samples 1164876 train steps 36090
Length 185
Episode 56501 total samples 1167416 train steps 36169
Length 226
Episode 56525 total samples 1169959 train steps 36249
Length 323
Episode 56548 total samples 1172516 train steps 36329
Length 139
Episode 56564 total samples 1175163 train steps 36411
Length 776
exploration 0.0999991000096
Episode 56586 total samples 1177796 train steps 36494
Length 273
Episode 56606 total samples 1180482 train steps 36578
Length 493
Episode 56629 total samples 1182990 train steps 36656
Length 138
Episode 56654 total samples 1185550 train steps 36736
Length 391
Episode 56678 total samples 1188096 train steps 36815
Length 169
Episode 56703 total samples 1190736 train steps 36898
Length 311
Episode 56724 total samples 1193421 train steps 36982
Length 547
Episode 56742 total samples 1195931 train steps 37060
Length 166
Episode 56762 total samples 1198676 train steps 37146
Length 594
Episode 56784 total samples 1201218 train steps 37226
Length 104
exploration 0.0999991000096
Episode 56805 total samples 1203906 train steps 37310
Length 409
Episode 56831 total samples 1206512 train steps 37391
Length 224
Episode 56853 total samples 1209017 train steps 37469
Length 79
Episode 56875 total samples 1211576 train steps 37549
Length 122
Episode 56898 total samples 1214211 train steps 37632
Length 303
Episode 56915 total samples 1216729 train steps 37710
Length 219
Episode 56939 total samples 1219307 train steps 37791
Length 391
Episode 56961 total samples 1221920 train steps 37872
Length 507
Episode 56989 total samples 1224512 train steps 37953
Length 274
exploration 0.0999991000096
Episode 57016 total samples 1227064 train steps 38033
Length 105
Episode 57040 total samples 1229572 train steps 38112
Length 111
Episode 57064 total samples 1232158 train steps 38192
Length 311
Episode 57093 total samples 1234763 train steps 38274
Length 241
Episode 57115 total samples 1237396 train steps 38356
Length 679
Episode 57138 total samples 1239966 train steps 38436
Length 221
Episode 57167 total samples 1242505 train steps 38516
Length 118
Episode 57192 total samples 1245116 train steps 38597
Length 225
Episode 57219 total samples 1247650 train steps 38677
Length 112
Episode 57241 total samples 1250217 train steps 38757
Length 225
exploration 0.0999991000096
Episode 57260 total samples 1252789 train steps 38837
Length 296
Episode 57281 total samples 1255457 train steps 38921
Length 354
Episode 57307 total samples 1258125 train steps 39004
Length 391
Episode 57328 total samples 1260701 train steps 39084
Length 327
Episode 57349 total samples 1263231 train steps 39163
Length 165
Episode 57366 total samples 1265777 train steps 39243
Length 249
Episode 57385 total samples 1268353 train steps 39324
Length 205
Episode 57408 total samples 1270983 train steps 39406
Length 315
Episode 57432 total samples 1273500 train steps 39484
Length 521
Episode 57460 total samples 1276081 train steps 39565
Length 253
exploration 0.0999991000096
Episode 57488 total samples 1278648 train steps 39645
Length 262
Episode 57512 total samples 1281249 train steps 39727
Length 408
Episode 57538 total samples 1284078 train steps 39815
Length 743
Episode 57561 total samples 1286598 train steps 39894
Length 41
Episode 57588 total samples 1289283 train steps 39978
Length 706
Episode 57614 total samples 1291954 train steps 40061
Length 598
Episode 57632 total samples 1294514 train steps 40141
Length 206
Episode 57651 total samples 1297270 train steps 40227
Length 917
Episode 57678 total samples 1300227 train steps 40320
Length 924
exploration 0.0999991000096
Episode 57706 total samples 1302764 train steps 40399
Length 218
Episode 57736 total samples 1305357 train steps 40480
Length 237
Episode 57766 total samples 1308000 train steps 40562
Length 350
Episode 57788 total samples 1310603 train steps 40644
Length 220
Episode 57811 total samples 1313177 train steps 40724
Length 328
Episode 57835 total samples 1315840 train steps 40807
Length 630
Episode 57854 total samples 1318375 train steps 40887
Length 160
Episode 57875 total samples 1320901 train steps 40966
Length 111
Episode 57895 total samples 1323480 train steps 41046
Length 211
Episode 57919 total samples 1325988 train steps 41125
Length 565
exploration 0.0999991000096
Episode 57942 total samples 1328551 train steps 41205
Length 166
Episode 57964 total samples 1331218 train steps 41288
Length 411
Episode 57984 total samples 1333721 train steps 41366
Length 418
Episode 58008 total samples 1336456 train steps 41452
Length 597
Episode 58025 total samples 1339013 train steps 41532
Length 218
Episode 58045 total samples 1341661 train steps 41614
Length 626
Episode 58066 total samples 1344185 train steps 41693
Length 222
Episode 58087 total samples 1346767 train steps 41774
Length 309
Episode 58109 total samples 1349308 train steps 41853
Length 133
Episode 58135 total samples 1351997 train steps 41937
Length 433
exploration 0.0999991000096
Episode 58152 total samples 1354513 train steps 42016
Length 183
Episode 58172 total samples 1357086 train steps 42096
Length 595
Episode 58194 total samples 1359660 train steps 42177
Length 452
Episode 58211 total samples 1362322 train steps 42260
Length 630
Episode 58229 total samples 1364853 train steps 42339
Length 274
Episode 58246 total samples 1367371 train steps 42418
Length 167
Episode 58262 total samples 1369912 train steps 42497
Length 426
Episode 58285 total samples 1372514 train steps 42579
Length 328
Episode 58305 total samples 1375062 train steps 42658
Length 1046
exploration 0.0999991000096
Episode 58330 total samples 1377593 train steps 42737
Length 76
Episode 58357 total samples 1380161 train steps 42818
Length 229
Episode 58384 total samples 1382724 train steps 42898
Length 514
Episode 58413 total samples 1385377 train steps 42981
Length 354
Episode 58442 total samples 1387986 train steps 43062
Length 218
Episode 58462 total samples 1390619 train steps 43144
Length 350
Episode 58485 total samples 1393188 train steps 43225
Length 354
Episode 58509 total samples 1395739 train steps 43304
Length 141
Episode 58532 total samples 1398346 train steps 43386
Length 251
Episode 58554 total samples 1400881 train steps 43465
Length 242
exploration 0.0999991000096
Episode 58571 total samples 1403444 train steps 43545
Length 192
Episode 58595 total samples 1405985 train steps 43625
Length 285
Episode 58617 total samples 1408494 train steps 43703
Length 367
Episode 58637 total samples 1411066 train steps 43783
Length 365
Episode 58664 total samples 1413684 train steps 43865
Length 262
Episode 58691 total samples 1416378 train steps 43949
Length 526
Episode 58717 total samples 1418981 train steps 44031
Length 845
Episode 58738 total samples 1421490 train steps 44109
Length 276
Episode 58770 total samples 1424056 train steps 44189
Length 211
Episode 58791 total samples 1426587 train steps 44268
Length 122
exploration 0.0999991000096
Episode 58807 total samples 1429175 train steps 44349
Length 356
Episode 58826 total samples 1431764 train steps 44430
Length 547
Episode 58850 total samples 1434349 train steps 44511
Length 316
Episode 58869 total samples 1436961 train steps 44593
Length 254
Episode 58887 total samples 1439596 train steps 44675
Length 308
Episode 58906 total samples 1442163 train steps 44755
Length 421
Episode 58930 total samples 1444741 train steps 44836
Length 278
Episode 58948 total samples 1447242 train steps 44914
Length 184
Episode 58977 total samples 1449783 train steps 44993
Length 85
Episode 59000 total samples 1452289 train steps 45072
Length 163
exploration 0.0999991000096
Episode 59025 total samples 1454848 train steps 45151
Length 195
Episode 59041 total samples 1457494 train steps 45234
Length 426
Episode 59067 total samples 1460017 train steps 45313
Length 204
Episode 59095 total samples 1462563 train steps 45393
Length 181
Episode 59116 total samples 1465280 train steps 45477
Length 500
Episode 59136 total samples 1467913 train steps 45560
Length 351
Episode 59163 total samples 1470438 train steps 45639
Length 116
Episode 59185 total samples 1473022 train steps 45719
Length 339
Episode 59208 total samples 1475721 train steps 45804
Length 455
exploration 0.0999991000096
Episode 59233 total samples 1478549 train steps 45892
Length 787
Episode 59253 total samples 1481121 train steps 45973
Length 182
Episode 59271 total samples 1483672 train steps 46052
Length 265
Episode 59291 total samples 1486233 train steps 46132
Length 372
Episode 59310 total samples 1488983 train steps 46218
Length 579
Episode 59334 total samples 1491541 train steps 46298
Length 314
Episode 59356 total samples 1494273 train steps 46384
Length 661
Episode 59379 total samples 1496852 train steps 46464
Length 224
Episode 59400 total samples 1499738 train steps 46554
Length 1069
Episode 59424 total samples 1502256 train steps 46633
Length 87
exploration 0.0999991000096
Episode 59447 total samples 1504766 train steps 46711
Length 198
Episode 59467 total samples 1507334 train steps 46792
Length 152
Episode 59490 total samples 1509863 train steps 46871
Length 106
Episode 59511 total samples 1512364 train steps 46949
Length 61
Episode 59530 total samples 1514967 train steps 47030
Length 309
Episode 59554 total samples 1517511 train steps 47110
Length 151
Episode 59572 total samples 1520100 train steps 47191
Length 225
Episode 59588 total samples 1522715 train steps 47272
Length 282
Episode 59607 total samples 1525230 train steps 47351
Length 297
Episode 59619 total samples 1527738 train steps 47429
Length 282
exploration 0.0999991000096
Episode 59638 total samples 1530278 train steps 47509
Length 382
Episode 59663 total samples 1532798 train steps 47587
Length 273
Episode 59679 total samples 1535633 train steps 47676
Length 773
Episode 59700 total samples 1538294 train steps 47759
Length 566
Episode 59725 total samples 1540869 train steps 47840
Length 426
Episode 59747 total samples 1543558 train steps 47924
Length 666
Episode 59766 total samples 1546241 train steps 48008
Length 611
Episode 59782 total samples 1548977 train steps 48093
Length 487
Episode 59806 total samples 1551534 train steps 48173
Length 469
exploration 0.0999991000096
Episode 59823 total samples 1554248 train steps 48258
Length 957
Episode 59843 total samples 1556882 train steps 48340
Length 366
Episode 59862 total samples 1559449 train steps 48420
Length 283
Episode 59886 total samples 1562016 train steps 48500
Length 143
Episode 59909 total samples 1564606 train steps 48581
Length 494
Episode 59926 total samples 1567359 train steps 48667
Length 586
Episode 59947 total samples 1569966 train steps 48749
Length 350
Episode 59968 total samples 1572602 train steps 48831
Length 362
Episode 59987 total samples 1575373 train steps 48918
Length 716
Episode 60009 total samples 1577943 train steps 48998
Length 326
exploration 0.0999991000096
Episode 60026 total samples 1580467 train steps 49077
Length 312
Episode 60048 total samples 1583169 train steps 49162
Length 463
Episode 60068 total samples 1585928 train steps 49248
Length 767
Episode 60083 total samples 1588485 train steps 49328
Length 253
Episode 60103 total samples 1591039 train steps 49407
Length 164
Episode 60127 total samples 1593723 train steps 49491
Length 586
Episode 60146 total samples 1596228 train steps 49570
Length 62
Episode 60163 total samples 1598862 train steps 49652
Length 687
Episode 60179 total samples 1601527 train steps 49735
Length 814
exploration 0.0999991000096
Episode 60195 total samples 1604228 train steps 49820
Length 726
Episode 60210 total samples 1606843 train steps 49901
Length 234
Episode 60231 total samples 1609489 train steps 49984
Length 489
Episode 60248 total samples 1612024 train steps 50063
Length 277
Episode 60267 total samples 1614625 train steps 50145
Length 322
Episode 60289 total samples 1617177 train steps 50224
Length 529
Episode 60307 total samples 1619772 train steps 50305
Length 499
Episode 60329 total samples 1622414 train steps 50388
Length 407
Episode 60347 total samples 1625068 train steps 50471
Length 473
Episode 60365 total samples 1627592 train steps 50550
Length 683
exploration 0.0999991000096
Episode 60389 total samples 1630142 train steps 50629
Length 171
Episode 60408 total samples 1632686 train steps 50709
Length 415
Episode 60433 total samples 1635494 train steps 50797
Length 622
Episode 60449 total samples 1638258 train steps 50883
Length 779
Episode 60467 total samples 1640918 train steps 50966
Length 383
Episode 60489 total samples 1643445 train steps 51045
Length 186
Episode 60511 total samples 1645965 train steps 51124
Length 139
Episode 60532 total samples 1648517 train steps 51204
Length 116
Episode 60548 total samples 1651043 train steps 51283
Length 126
exploration 0.0999991000096
Episode 60562 total samples 1653644 train steps 51364
Length 365
Episode 60583 total samples 1656252 train steps 51445
Length 288
Episode 60600 total samples 1658868 train steps 51527
Length 357
Episode 60619 total samples 1661388 train steps 51606
Length 348
Episode 60636 total samples 1664010 train steps 51688
Length 374
Episode 60657 total samples 1666766 train steps 51774
Length 517
Episode 60677 total samples 1669434 train steps 51857
Length 519
Episode 60696 total samples 1672145 train steps 51942
Length 447
Episode 60716 total samples 1674727 train steps 52023
Length 169
Episode 60733 total samples 1677263 train steps 52102
Length 402
exploration 0.0999991000096
Episode 60755 total samples 1679838 train steps 52182
Length 377
Episode 60774 total samples 1682443 train steps 52264
Length 387
Episode 60793 total samples 1684947 train steps 52342
Length 166
Episode 60814 total samples 1687525 train steps 52423
Length 213
Episode 60833 total samples 1690243 train steps 52508
Length 533
Episode 60852 total samples 1692748 train steps 52586
Length 29
Episode 60867 total samples 1695296 train steps 52665
Length 168
Episode 60890 total samples 1697822 train steps 52744
Length 56
Episode 60910 total samples 1700399 train steps 52825
Length 511
Episode 60929 total samples 1702932 train steps 52904
Length 274
exploration 0.0999991000096
Episode 60942 total samples 1705485 train steps 52984
Length 696
Episode 60963 total samples 1708027 train steps 53063
Length 344
Episode 60983 total samples 1710671 train steps 53146
Length 315
Episode 61001 total samples 1713200 train steps 53225
Length 360
Episode 61022 total samples 1715774 train steps 53305
Length 284
Episode 61041 total samples 1718320 train steps 53385
Length 289
Episode 61059 total samples 1720979 train steps 53468
Length 576
Episode 61077 total samples 1723529 train steps 53548
Length 376
Episode 61096 total samples 1726159 train steps 53630
Length 333
exploration 0.0999991000096
Episode 61120 total samples 1728833 train steps 53714
Length 380
Episode 61137 total samples 1731397 train steps 53794
Length 162
Episode 61154 total samples 1733936 train steps 53873
Length 474
Episode 61178 total samples 1736490 train steps 53953
Length 335
Episode 61197 total samples 1739185 train steps 54037
Length 483
Episode 61214 total samples 1741724 train steps 54116
Length 188
Episode 61237 total samples 1744516 train steps 54204
Length 727
Episode 61255 total samples 1747240 train steps 54289
Length 871
Episode 61271 total samples 1749813 train steps 54369
Length 373
Episode 61293 total samples 1752347 train steps 54448
Length 392
exploration 0.0999991000096
Episode 61312 total samples 1754980 train steps 54531
Length 343
Episode 61328 total samples 1757483 train steps 54609
Length 497
Episode 61348 total samples 1760009 train steps 54688
Length 236
Episode 61367 total samples 1762600 train steps 54769
Length 869
Episode 61384 total samples 1765146 train steps 54848
Length 311
Episode 61400 total samples 1767713 train steps 54929
Length 382
Episode 61419 total samples 1770340 train steps 55011
Length 294
Episode 61438 total samples 1772963 train steps 55093
Length 815
Episode 61451 total samples 1775523 train steps 55173
Length 462
Episode 61469 total samples 1778100 train steps 55253
Length 320
exploration 0.0999991000096
Episode 61488 total samples 1780709 train steps 55335
Length 701
Episode 61502 total samples 1783212 train steps 55413
Length 1177
Episode 61522 total samples 1786078 train steps 55502
Length 1022
Episode 61539 total samples 1788719 train steps 55585
Length 309
Episode 61561 total samples 1791571 train steps 55674
Length 908
Episode 61576 total samples 1794262 train steps 55758
Length 496
Episode 61592 total samples 1796919 train steps 55841
Length 784
Episode 61606 total samples 1799562 train steps 55924
Length 888
Episode 61621 total samples 1802144 train steps 56004
Length 284
exploration 0.0999991000096
Episode 61645 total samples 1804922 train steps 56091
Length 653
Episode 61670 total samples 1807451 train steps 56170
Length 268
Episode 61692 total samples 1810116 train steps 56254
Length 355
Episode 61707 total samples 1812841 train steps 56339
Length 581
Episode 61722 total samples 1815463 train steps 56421
Length 253
Episode 61738 total samples 1818034 train steps 56501
Length 317
Episode 61756 total samples 1820892 train steps 56590
Length 770
Episode 61769 total samples 1823399 train steps 56669
Length 100
Episode 61780 total samples 1825963 train steps 56749
Length 537
Episode 61798 total samples 1828476 train steps 56827
Length 134
exploration 0.0999991000096
Episode 61815 total samples 1831103 train steps 56909
Length 300
Episode 61833 total samples 1833615 train steps 56988
Length 31
Episode 61850 total samples 1836193 train steps 57069
Length 325
Episode 61875 total samples 1838748 train steps 57148
Length 135
Episode 61894 total samples 1841786 train steps 57243
Length 1587
Episode 61911 total samples 1844350 train steps 57323
Length 151
Episode 61925 total samples 1847086 train steps 57409
Length 776
Episode 61948 total samples 1849734 train steps 57492
Length 394
Episode 61962 total samples 1852428 train steps 57576
Length 705
exploration 0.0999991000096
Episode 61977 total samples 1855087 train steps 57659
Length 531
Episode 61995 total samples 1857662 train steps 57739
Length 211
Episode 62011 total samples 1860236 train steps 57820
Length 897
Episode 62025 total samples 1862793 train steps 57900
Length 292
Episode 62035 total samples 1865571 train steps 57987
Length 991
Episode 62056 total samples 1868248 train steps 58070
Length 375
Episode 62264 total samples 1870752 train steps 58148
Length 10
Episode 62744 total samples 1873254 train steps 58227
Length 10
Episode 63131 total samples 1876016 train steps 58313
Length 684
Episode 63150 total samples 1878786 train steps 58400
Length 737
exploration 0.0999991000096
Episode 63169 total samples 1881433 train steps 58482
Length 387
Episode 63184 total samples 1884032 train steps 58563
Length 618
Episode 63203 total samples 1886634 train steps 58645
Length 209
Episode 63223 total samples 1889308 train steps 58728
Length 504
Episode 63240 total samples 1892209 train steps 58819
Length 1038
Episode 63253 total samples 1894821 train steps 58901
Length 482
Episode 63267 total samples 1897384 train steps 58981
Length 533
Episode 63287 total samples 1900051 train steps 59064
Length 413
Episode 63303 total samples 1902577 train steps 59143
Length 56
exploration 0.0999991000096
Episode 63325 total samples 1905116 train steps 59222
Length 484
Episode 63346 total samples 1907780 train steps 59306
Length 431
Episode 63359 total samples 1910463 train steps 59389
Length 1197
Episode 63375 total samples 1913207 train steps 59475
Length 540
Episode 63386 total samples 1915779 train steps 59556
Length 161
Episode 63404 total samples 1918399 train steps 59637
Length 695
Episode 63416 total samples 1920991 train steps 59718
Length 432
Episode 63442 total samples 1923554 train steps 59799
Length 539
Episode 63462 total samples 1926249 train steps 59883
Length 802
Episode 63488 total samples 1929030 train steps 59970
Length 652
exploration 0.0999991000096
Episode 63506 total samples 1931571 train steps 60049
Length 186
Episode 63518 total samples 1934111 train steps 60128
Length 468
Episode 63531 total samples 1936703 train steps 60209
Length 223
Episode 63552 total samples 1939364 train steps 60293
Length 324
Episode 63571 total samples 1942091 train steps 60378
Length 595
Episode 63591 total samples 1944602 train steps 60456
Length 105
Episode 63607 total samples 1947154 train steps 60536
Length 930
Episode 63622 total samples 1949665 train steps 60615
Length 590
Episode 63634 total samples 1952415 train steps 60700
Length 788
exploration 0.0999991000096
Episode 63649 total samples 1954933 train steps 60779
Length 58
Episode 63665 total samples 1957454 train steps 60858
Length 77
Episode 63687 total samples 1960080 train steps 60940
Length 268
Episode 63708 total samples 1962646 train steps 61020
Length 393
Episode 63718 total samples 1965178 train steps 61099
Length 668
Episode 63738 total samples 1967769 train steps 61180
Length 219
Episode 63759 total samples 1970496 train steps 61265
Length 488
Episode 63775 total samples 1973179 train steps 61349
Length 1025
Episode 63791 total samples 1975730 train steps 61429
Length 350
Episode 63808 total samples 1978258 train steps 61508
Length 67
exploration 0.0999991000096
Episode 63824 total samples 1980813 train steps 61588
Length 374
Episode 63842 total samples 1983662 train steps 61677
Length 862
Episode 63856 total samples 1986281 train steps 61759
Length 467
Episode 63867 total samples 1989096 train steps 61847
Length 749
Episode 63887 total samples 1991612 train steps 61925
Length 32
Episode 63912 total samples 1994209 train steps 62007
Length 200
Episode 63934 total samples 1996751 train steps 62086
Length 111
Episode 63949 total samples 1999502 train steps 62172
Length 602
Episode 63969 total samples 2002033 train steps 62251
Length 201
Episode 63992 total samples 2004592 train steps 62331
Length 229
exploration 0.0999991000096
Episode 64018 total samples 2007095 train steps 62409
Length 32
Episode 64044 total samples 2009658 train steps 62489
Length 371
Episode 64065 total samples 2012163 train steps 62568
Length 241
Episode 64079 total samples 2014971 train steps 62655
Length 828
Episode 64099 total samples 2017838 train steps 62745
Length 804
Episode 64111 total samples 2020688 train steps 62834
Length 797
Episode 64127 total samples 2023302 train steps 62916
Length 328
Episode 64142 total samples 2025858 train steps 62996
Length 380
Episode 64163 total samples 2028493 train steps 63078
Length 661
exploration 0.0999991000096
Episode 64179 total samples 2031106 train steps 63160
Length 263
Episode 64194 total samples 2033759 train steps 63242
Length 401
Episode 64217 total samples 2036388 train steps 63325
Length 706
Episode 64232 total samples 2039024 train steps 63407
Length 1238
Episode 64249 total samples 2041575 train steps 63487
Length 111
Episode 64265 total samples 2044168 train steps 63568
Length 386
Episode 64278 total samples 2046746 train steps 63648
Length 860
Episode 64294 total samples 2049248 train steps 63726
Length 143
Episode 64311 total samples 2051862 train steps 63808
Length 555
Episode 64325 total samples 2054897 train steps 63903
Length 1370
exploration 0.0999991000096
Episode 64352 total samples 2057451 train steps 63983
Length 146
Episode 64373 total samples 2060225 train steps 64070
Length 626
Episode 64388 total samples 2062827 train steps 64151
Length 498
Episode 64402 total samples 2065376 train steps 64230
Length 896
Episode 64423 total samples 2068162 train steps 64318
Length 584
Episode 64436 total samples 2070798 train steps 64400
Length 332
Episode 64458 total samples 2073385 train steps 64481
Length 378
Episode 64473 total samples 2076022 train steps 64563
Length 507
Episode 64490 total samples 2078752 train steps 64648
Length 661
exploration 0.0999991000096
Episode 64502 total samples 2081305 train steps 64728
Length 702
Episode 64516 total samples 2083829 train steps 64807
Length 132
Episode 64536 total samples 2086580 train steps 64893
Length 519
Episode 64555 total samples 2089091 train steps 64972
Length 152
Episode 64574 total samples 2091694 train steps 65053
Length 322
Episode 64591 total samples 2094353 train steps 65136
Length 718
Episode 64604 total samples 2096856 train steps 65214
Length 324
Episode 64622 total samples 2099406 train steps 65294
Length 137
Episode 64646 total samples 2101947 train steps 65373
Length 185
Episode 64666 total samples 2105043 train steps 65470
Length 1294
exploration 0.0999991000096
Episode 64681 total samples 2107594 train steps 65550
Length 220
Episode 64698 total samples 2110150 train steps 65630
Length 557
Episode 64712 total samples 2112701 train steps 65709
Length 260
Episode 64735 total samples 2115396 train steps 65794
Length 580
Episode 64752 total samples 2117928 train steps 65873
Length 153
Episode 64769 total samples 2120510 train steps 65953
Length 242
Episode 64789 total samples 2123057 train steps 66033
Length 244
Episode 64807 total samples 2125678 train steps 66115
Length 1232
Episode 64825 total samples 2128247 train steps 66195
Length 188
exploration 0.0999991000096
Episode 64842 total samples 2130864 train steps 66277
Length 504
Episode 64864 total samples 2133485 train steps 66359
Length 288
Episode 64881 total samples 2136014 train steps 66438
Length 345
Episode 64897 total samples 2138601 train steps 66519
Length 454
Episode 64913 total samples 2141482 train steps 66609
Length 794
Episode 64939 total samples 2144014 train steps 66688
Length 106
Episode 64952 total samples 2146738 train steps 66773
Length 764
Episode 64975 total samples 2149251 train steps 66852
Length 172
Episode 64998 total samples 2152210 train steps 66944
Length 1208
Episode 65020 total samples 2154842 train steps 67026
Length 399
exploration 0.0999991000096
Episode 65038 total samples 2157377 train steps 67106
Length 552
Episode 65059 total samples 2159894 train steps 67184
Length 1067
Episode 65072 total samples 2162499 train steps 67266
Length 374
Episode 65093 total samples 2165032 train steps 67345
Length 334
Episode 65113 total samples 2167637 train steps 67426
Length 377
Episode 65128 total samples 2170156 train steps 67505
Length 98
Episode 65143 total samples 2172668 train steps 67583
Length 158
Episode 65158 total samples 2175269 train steps 67665
Length 415
Episode 65179 total samples 2177818 train steps 67744
Length 552
Episode 65197 total samples 2180393 train steps 67825
Length 160
exploration 0.0999991000096
Episode 65213 total samples 2182926 train steps 67904
Length 299
Episode 65222 total samples 2185453 train steps 67983
Length 545
Episode 65239 total samples 2187965 train steps 68061
Length 451
Episode 65258 total samples 2190547 train steps 68142
Length 256
Episode 65270 total samples 2193085 train steps 68221
Length 114
Episode 65285 total samples 2195667 train steps 68302
Length 621
Episode 65299 total samples 2198190 train steps 68381
Length 1716
Episode 65320 total samples 2200772 train steps 68462
Length 246
Episode 65337 total samples 2203298 train steps 68541
Length 319
Episode 65347 total samples 2205895 train steps 68622
Length 294
exploration 0.0999991000096
Episode 65360 total samples 2208525 train steps 68704
Length 415
Episode 65379 total samples 2211579 train steps 68799
Length 1318
Episode 65397 total samples 2214349 train steps 68886
Length 777
Episode 65419 total samples 2216966 train steps 68968
Length 310
Episode 65431 total samples 2219523 train steps 69048
Length 123
Episode 65445 total samples 2222278 train steps 69134
Length 608
Episode 65467 total samples 2224799 train steps 69212
Length 44
Episode 65475 total samples 2227511 train steps 69297
Length 483
Episode 65493 total samples 2230026 train steps 69376
Length 127
exploration 0.0999991000096
Episode 65516 total samples 2232602 train steps 69456
Length 375
Episode 65533 total samples 2235161 train steps 69536
Length 353
Episode 65547 total samples 2237785 train steps 69618
Length 499
Episode 65567 total samples 2240340 train steps 69698
Length 179
Episode 65581 total samples 2242995 train steps 69781
Length 390
Episode 65604 total samples 2245705 train steps 69866
Length 450
Episode 65629 total samples 2248630 train steps 69957
Length 880
Episode 65647 total samples 2251368 train steps 70043
Length 806
Episode 65662 total samples 2253964 train steps 70124
Length 526
exploration 0.0999991000096
Episode 65678 total samples 2256494 train steps 70203
Length 372
Episode 65694 total samples 2259199 train steps 70287
Length 429
Episode 65711 total samples 2262139 train steps 70379
Length 961
Episode 65720 total samples 2264675 train steps 70459
Length 1119
Episode 65732 total samples 2267405 train steps 70544
Length 1006
Episode 65750 total samples 2269912 train steps 70622
Length 276
Episode 65767 total samples 2272766 train steps 70711
Length 824
Episode 65777 total samples 2275422 train steps 70794
Length 741
Episode 65788 total samples 2277961 train steps 70874
Length 933
Episode 65803 total samples 2280598 train steps 70956
Length 348
exploration 0.0999991000096
Episode 65817 total samples 2283280 train steps 71040
Length 451
Episode 65828 total samples 2285842 train steps 71120
Length 216
Episode 65841 total samples 2288418 train steps 71201
Length 953
Episode 65859 total samples 2290936 train steps 71279
Length 249
Episode 65878 total samples 2293662 train steps 71364
Length 615
Episode 65898 total samples 2296200 train steps 71444
Length 522
Episode 65914 total samples 2298757 train steps 71524
Length 395
Episode 65930 total samples 2301334 train steps 71604
Length 1035
Episode 65941 total samples 2304049 train steps 71689
Length 571
Episode 65949 total samples 2306550 train steps 71767
Length 1592
exploration 0.0999991000096
Episode 65962 total samples 2309428 train steps 71857
Length 837
Episode 65979 total samples 2312807 train steps 71963
Length 2493
Episode 65995 total samples 2315433 train steps 72045
Length 661
Episode 66008 total samples 2318200 train steps 72131
Length 1512
Episode 66023 total samples 2320744 train steps 72211
Length 816
Episode 66034 total samples 2323278 train steps 72290
Length 801
Episode 66044 total samples 2325857 train steps 72371
Length 1265
Episode 66054 total samples 2328792 train steps 72462
Length 1063
Episode 66482 total samples 2331293 train steps 72540
Length 11
exploration 0.0999991000096
Episode 66964 total samples 2333794 train steps 72619
Length 8
Episode 67448 total samples 2336295 train steps 72697
Length 9
Episode 67932 total samples 2338798 train steps 72775
Length 9
Episode 68413 total samples 2341301 train steps 72853
Length 8
Episode 68890 total samples 2343802 train steps 72931
Length 10
Episode 69377 total samples 2346303 train steps 73009
Length 9
Episode 69857 total samples 2348804 train steps 73088
Length 10
Episode 70343 total samples 2351305 train steps 73166
Length 9
Episode 70818 total samples 2353809 train steps 73244
Length 11
Episode 71298 total samples 2356314 train steps 73322
Length 9
exploration 0.0999991000096
Episode 71782 total samples 2358817 train steps 73401
Length 9
Episode 72266 total samples 2361321 train steps 73479
Length 13
Episode 72749 total samples 2363822 train steps 73557
Length 10
Episode 73229 total samples 2366327 train steps 73635
Length 9
Episode 73709 total samples 2368832 train steps 73713
Length 10
Episode 74184 total samples 2371334 train steps 73792
Length 10
Episode 74662 total samples 2373836 train steps 73870
Length 9
Episode 75143 total samples 2376340 train steps 73948
Length 9
Episode 75618 total samples 2378842 train steps 74026
Length 9
Episode 76090 total samples 2381345 train steps 74105
Length 8
exploration 0.0999991000096
Episode 76571 total samples 2383850 train steps 74183
Length 9
Episode 77044 total samples 2386356 train steps 74261
Length 12
Episode 77525 total samples 2388861 train steps 74339
Length 9
Episode 78007 total samples 2391365 train steps 74418
Length 9
Episode 78488 total samples 2393869 train steps 74496
Length 10
Episode 78965 total samples 2396372 train steps 74574
Length 9
Episode 79443 total samples 2398876 train steps 74652
Length 12
Episode 79923 total samples 2401380 train steps 74731
Length 21
Episode 80399 total samples 2403884 train steps 74809
Length 9
Episode 80876 total samples 2406387 train steps 74887
Length 13
exploration 0.0999991000096
Episode 81358 total samples 2408891 train steps 74965
Length 8
Episode 81825 total samples 2411395 train steps 75044
Length 9
Episode 82305 total samples 2413900 train steps 75122
Length 10
Episode 82783 total samples 2416401 train steps 75200
Length 9
Episode 83265 total samples 2418902 train steps 75278
Length 8
Episode 83739 total samples 2421406 train steps 75356
Length 9
Episode 84221 total samples 2423907 train steps 75435
Length 8
Episode 84701 total samples 2426410 train steps 75513
Length 11
Episode 85175 total samples 2428911 train steps 75591
Length 10
Episode 85648 total samples 2431417 train steps 75669
Length 23
exploration 0.0999991000096
Episode 86132 total samples 2433918 train steps 75747
Length 9
Episode 86614 total samples 2436420 train steps 75826
Length 9
Episode 87096 total samples 2438922 train steps 75904
Length 10
Episode 87578 total samples 2441426 train steps 75982
Length 9
Episode 88060 total samples 2443930 train steps 76060
Length 9
Episode 88535 total samples 2446432 train steps 76138
Length 8
Episode 89016 total samples 2448933 train steps 76217
Length 9
Episode 89495 total samples 2451436 train steps 76295
Length 9
Episode 89977 total samples 2453941 train steps 76373
Length 10
Episode 90452 total samples 2456443 train steps 76451
Length 9
exploration 0.0999991000096
Episode 90925 total samples 2458947 train steps 76530
Length 10
Episode 91406 total samples 2461448 train steps 76608
Length 9
Episode 91885 total samples 2463952 train steps 76686
Length 9
Episode 92364 total samples 2466454 train steps 76764
Length 8
Episode 92843 total samples 2468956 train steps 76842
Length 9
Episode 93313 total samples 2471457 train steps 76921
Length 10
Episode 93793 total samples 2473961 train steps 76999
Length 8
Episode 94272 total samples 2476464 train steps 77077
Length 9
Episode 94752 total samples 2478969 train steps 77155
Length 10
Episode 95235 total samples 2481472 train steps 77233
Length 9
exploration 0.0999991000096
Episode 95721 total samples 2483974 train steps 77312
Length 8
Episode 96197 total samples 2486480 train steps 77390
Length 11
Episode 96676 total samples 2488984 train steps 77468
Length 9
Episode 97156 total samples 2491488 train steps 77546
Length 10
Episode 97632 total samples 2493996 train steps 77625
Length 18
Episode 98105 total samples 2496499 train steps 77703
Length 10
Episode 98583 total samples 2499000 train steps 77781
Length 10
Episode 99063 total samples 2501502 train steps 77859
Length 8
Episode 99535 total samples 2504004 train steps 77938
Length 10
plot not available. Move on.
In [5]:
car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
Episode 477 total samples 2508919 train steps 78091
Length 14
Episode 951 total samples 2511421 train steps 78169
Length 8
Episode 1436 total samples 2513924 train steps 78248
Length 10
Episode 1912 total samples 2516430 train steps 78326
Length 14
Episode 2386 total samples 2518931 train steps 78404
Length 10
Episode 2867 total samples 2521433 train steps 78482
Length 9
Episode 3348 total samples 2523936 train steps 78560
Length 8
Episode 3827 total samples 2526438 train steps 78639
Length 10
Episode 4304 total samples 2528942 train steps 78717
Length 9
exploration 0.0999991000096
Episode 4779 total samples 2531445 train steps 78795
Length 8
Episode 5258 total samples 2533950 train steps 78873
Length 11
Episode 5740 total samples 2536455 train steps 78952
Length 10
Episode 6217 total samples 2538959 train steps 79030
Length 8
Episode 6696 total samples 2541463 train steps 79108
Length 10
Episode 7178 total samples 2543967 train steps 79186
Length 10
Episode 7656 total samples 2546469 train steps 79265
Length 9
Episode 8134 total samples 2548972 train steps 79343
Length 8
Episode 8612 total samples 2551478 train steps 79421
Length 13
Episode 9087 total samples 2553981 train steps 79499
Length 11
exploration 0.0999991000096
Episode 9564 total samples 2556486 train steps 79578
Length 9
Episode 10040 total samples 2558988 train steps 79656
Length 9
Episode 10517 total samples 2561490 train steps 79734
Length 8
Episode 10993 total samples 2563993 train steps 79812
Length 10
Episode 11469 total samples 2566497 train steps 79891
Length 14
Episode 11944 total samples 2568999 train steps 79969
Length 9
Episode 12423 total samples 2571501 train steps 80047
Length 10
Episode 12894 total samples 2574004 train steps 80125
Length 10
Episode 13367 total samples 2576509 train steps 80203
Length 10
Episode 13850 total samples 2579010 train steps 80282
Length 10
exploration 0.0999991000096
Episode 14334 total samples 2581516 train steps 80360
Length 12
Episode 14817 total samples 2584017 train steps 80438
Length 8
Episode 15300 total samples 2586519 train steps 80516
Length 9
Episode 15775 total samples 2589021 train steps 80594
Length 9
Episode 16253 total samples 2591522 train steps 80673
Length 10
Episode 16732 total samples 2594023 train steps 80751
Length 12
Episode 17215 total samples 2596529 train steps 80829
Length 11
Episode 17690 total samples 2599032 train steps 80907
Length 9
Episode 18165 total samples 2601537 train steps 80986
Length 10
Episode 18642 total samples 2604039 train steps 81064
Length 9
exploration 0.0999991000096
Episode 19115 total samples 2606541 train steps 81142
Length 8
Episode 19593 total samples 2609046 train steps 81220
Length 10
Episode 20063 total samples 2611548 train steps 81298
Length 10
Episode 20539 total samples 2614053 train steps 81377
Length 11
Episode 21012 total samples 2616555 train steps 81455
Length 14
Episode 21488 total samples 2619056 train steps 81533
Length 10
Episode 21963 total samples 2621560 train steps 81611
Length 9
Episode 22436 total samples 2624066 train steps 81690
Length 22
Episode 22914 total samples 2626568 train steps 81768
Length 11
Episode 23391 total samples 2629072 train steps 81846
Length 8
exploration 0.0999991000096
Episode 23868 total samples 2631575 train steps 81924
Length 10
Episode 24348 total samples 2634076 train steps 82002
Length 10
Episode 24826 total samples 2636578 train steps 82081
Length 10
Episode 25305 total samples 2639081 train steps 82159
Length 9
Episode 25784 total samples 2641584 train steps 82237
Length 8
Episode 26257 total samples 2644088 train steps 82315
Length 9
Episode 26740 total samples 2646593 train steps 82394
Length 11
Episode 27223 total samples 2649096 train steps 82472
Length 9
Episode 27705 total samples 2651599 train steps 82550
Length 9
Episode 28179 total samples 2654100 train steps 82628
Length 10
exploration 0.0999991000096
Episode 28655 total samples 2656602 train steps 82706
Length 9
Episode 29137 total samples 2659106 train steps 82785
Length 21
Episode 29622 total samples 2661607 train steps 82863
Length 10
Episode 30101 total samples 2664111 train steps 82941
Length 13
Episode 30574 total samples 2666612 train steps 83019
Length 8
Episode 31044 total samples 2669117 train steps 83097
Length 12
Episode 31525 total samples 2671618 train steps 83176
Length 9
Episode 32005 total samples 2674123 train steps 83254
Length 9
Episode 32479 total samples 2676626 train steps 83332
Length 10
Episode 32958 total samples 2679130 train steps 83410
Length 9
exploration 0.0999991000096
Episode 33431 total samples 2681635 train steps 83489
Length 9
Episode 33915 total samples 2684137 train steps 83567
Length 10
Episode 34396 total samples 2686641 train steps 83645
Length 10
Episode 34878 total samples 2689145 train steps 83723
Length 9
Episode 35364 total samples 2691646 train steps 83801
Length 9
Episode 35841 total samples 2694148 train steps 83880
Length 15
Episode 36310 total samples 2696651 train steps 83958
Length 21
Episode 36792 total samples 2699153 train steps 84036
Length 15
Episode 37273 total samples 2701656 train steps 84114
Length 8
Episode 37756 total samples 2704157 train steps 84192
Length 10
exploration 0.0999991000096
Episode 38235 total samples 2706658 train steps 84271
Length 10
Episode 38715 total samples 2709162 train steps 84349
Length 9
Episode 39190 total samples 2711663 train steps 84427
Length 9
Episode 39673 total samples 2714165 train steps 84505
Length 9
Episode 40153 total samples 2716670 train steps 84583
Length 9
Episode 40636 total samples 2719172 train steps 84662
Length 10
Episode 41114 total samples 2721677 train steps 84740
Length 11
Episode 41595 total samples 2724182 train steps 84818
Length 9
Episode 42080 total samples 2726684 train steps 84896
Length 14
Episode 42558 total samples 2729188 train steps 84975
Length 10
exploration 0.0999991000096
Episode 43035 total samples 2731690 train steps 85053
Length 9
Episode 43514 total samples 2734194 train steps 85131
Length 9
Episode 43992 total samples 2736695 train steps 85209
Length 11
Episode 44473 total samples 2739198 train steps 85287
Length 9
Episode 44954 total samples 2741703 train steps 85366
Length 12
Episode 45442 total samples 2744206 train steps 85444
Length 10
Episode 45918 total samples 2746712 train steps 85522
Length 13
Episode 46399 total samples 2749214 train steps 85600
Length 10
Episode 46881 total samples 2751719 train steps 85679
Length 10
Episode 47367 total samples 2754223 train steps 85757
Length 16
exploration 0.0999991000096
Episode 47836 total samples 2756726 train steps 85835
Length 8
Episode 48315 total samples 2759229 train steps 85913
Length 9
Episode 48788 total samples 2761734 train steps 85992
Length 17
Episode 49259 total samples 2764237 train steps 86070
Length 14
Episode 49736 total samples 2766738 train steps 86148
Length 9
Episode 50210 total samples 2769242 train steps 86226
Length 10
Episode 50688 total samples 2771746 train steps 86305
Length 10
Episode 51171 total samples 2774247 train steps 86383
Length 16
Episode 51656 total samples 2776750 train steps 86461
Length 10
Episode 52136 total samples 2779255 train steps 86539
Length 10
exploration 0.0999991000096
Episode 52612 total samples 2781758 train steps 86617
Length 11
Episode 53093 total samples 2784259 train steps 86696
Length 8
Episode 53573 total samples 2786762 train steps 86774
Length 9
Episode 54049 total samples 2789264 train steps 86852
Length 10
Episode 54529 total samples 2791767 train steps 86930
Length 11
Episode 55007 total samples 2794270 train steps 87008
Length 9
Episode 55489 total samples 2796771 train steps 87087
Length 9
Episode 55969 total samples 2799273 train steps 87165
Length 10
Episode 56441 total samples 2801775 train steps 87243
Length 9
Episode 56923 total samples 2804276 train steps 87321
Length 9
exploration 0.0999991000096
Episode 57402 total samples 2806778 train steps 87399
Length 13
Episode 57884 total samples 2809279 train steps 87477
Length 10
Episode 58360 total samples 2811784 train steps 87556
Length 10
Episode 58842 total samples 2814289 train steps 87634
Length 9
Episode 59318 total samples 2816793 train steps 87712
Length 10
Episode 59786 total samples 2819295 train steps 87790
Length 9
Episode 60259 total samples 2821796 train steps 87869
Length 14
Episode 60731 total samples 2824299 train steps 87947
Length 10
Episode 61205 total samples 2826802 train steps 88025
Length 9
Episode 61685 total samples 2829307 train steps 88103
Length 9
exploration 0.0999991000096
Episode 62174 total samples 2831810 train steps 88182
Length 9
Episode 62651 total samples 2834315 train steps 88260
Length 9
Episode 63127 total samples 2836819 train steps 88338
Length 10
Episode 63600 total samples 2839320 train steps 88416
Length 9
Episode 64075 total samples 2841826 train steps 88495
Length 17
Episode 64556 total samples 2844329 train steps 88573
Length 9
Episode 65033 total samples 2846831 train steps 88651
Length 9
Episode 65506 total samples 2849334 train steps 88729
Length 11
Episode 65978 total samples 2851835 train steps 88807
Length 13
Episode 66462 total samples 2854339 train steps 88886
Length 12
exploration 0.0999991000096
Episode 66946 total samples 2856843 train steps 88964
Length 12
Episode 67430 total samples 2859345 train steps 89042
Length 9
Episode 67909 total samples 2861849 train steps 89120
Length 10
Episode 68384 total samples 2864354 train steps 89199
Length 13
Episode 68866 total samples 2866858 train steps 89277
Length 10
Episode 69347 total samples 2869361 train steps 89355
Length 10
Episode 69824 total samples 2871866 train steps 89433
Length 12
Episode 70307 total samples 2874370 train steps 89512
Length 11
Episode 70783 total samples 2876873 train steps 89590
Length 8
Episode 71262 total samples 2879374 train steps 89668
Length 9
exploration 0.0999991000096
Episode 71734 total samples 2881877 train steps 89746
Length 10
Episode 72213 total samples 2884378 train steps 89824
Length 9
Episode 72690 total samples 2886883 train steps 89903
Length 14
Episode 73176 total samples 2889387 train steps 89981
Length 10
Episode 73656 total samples 2891889 train steps 90059
Length 9
Episode 74136 total samples 2894394 train steps 90137
Length 10
Episode 74618 total samples 2896897 train steps 90216
Length 10
Episode 75097 total samples 2899400 train steps 90294
Length 8
Episode 75575 total samples 2901902 train steps 90372
Length 10
Episode 76052 total samples 2904406 train steps 90450
Length 9
exploration 0.0999991000096
Episode 76532 total samples 2906909 train steps 90528
Length 10
Episode 77009 total samples 2909411 train steps 90607
Length 10
Episode 77489 total samples 2911916 train steps 90685
Length 10
Episode 77963 total samples 2914420 train steps 90763
Length 9
Episode 78441 total samples 2916925 train steps 90841
Length 9
Episode 78911 total samples 2919430 train steps 90920
Length 10
Episode 79384 total samples 2921932 train steps 90998
Length 10
Episode 79861 total samples 2924433 train steps 91076
Length 9
Episode 80340 total samples 2926935 train steps 91154
Length 11
Episode 80824 total samples 2929439 train steps 91232
Length 15
exploration 0.0999991000096
Episode 81300 total samples 2931941 train steps 91311
Length 10
Episode 81781 total samples 2934449 train steps 91389
Length 18
Episode 82263 total samples 2936950 train steps 91467
Length 9
Episode 82735 total samples 2939454 train steps 91545
Length 9
Episode 83210 total samples 2941959 train steps 91624
Length 9
Episode 83687 total samples 2944462 train steps 91702
Length 9
Episode 84162 total samples 2946965 train steps 91780
Length 9
Episode 84638 total samples 2949469 train steps 91858
Length 10
Episode 85111 total samples 2951972 train steps 91937
Length 10
Episode 85589 total samples 2954475 train steps 92015
Length 10
exploration 0.0999991000096
Episode 86062 total samples 2956976 train steps 92093
Length 9
Episode 86544 total samples 2959478 train steps 92171
Length 9
Episode 87022 total samples 2961979 train steps 92249
Length 10
Episode 87492 total samples 2964482 train steps 92328
Length 9
Episode 87976 total samples 2966984 train steps 92406
Length 10
Episode 88454 total samples 2969488 train steps 92484
Length 9
Episode 88930 total samples 2971991 train steps 92562
Length 9
Episode 89416 total samples 2974495 train steps 92640
Length 13
Episode 89900 total samples 2976998 train steps 92719
Length 8
Episode 90380 total samples 2979499 train steps 92797
Length 10
exploration 0.0999991000096
Episode 90856 total samples 2982000 train steps 92875
Length 9
Episode 91339 total samples 2984501 train steps 92953
Length 9
Episode 91819 total samples 2987006 train steps 93031
Length 10
Episode 92300 total samples 2989507 train steps 93110
Length 8
Episode 92777 total samples 2992009 train steps 93188
Length 10
Episode 93255 total samples 2994513 train steps 93266
Length 10
Episode 93734 total samples 2997016 train steps 93344
Length 10
Episode 94213 total samples 2999521 train steps 93423
Length 10
Episode 94691 total samples 3002025 train steps 93501
Length 19
Episode 95174 total samples 3004529 train steps 93579
Length 10
exploration 0.0999991000096
Episode 95658 total samples 3007035 train steps 93657
Length 11
Episode 96136 total samples 3009536 train steps 93735
Length 9
Episode 96616 total samples 3012041 train steps 93814
Length 9
Episode 97096 total samples 3014546 train steps 93892
Length 12
Episode 97569 total samples 3017051 train steps 93970
Length 11
Episode 98043 total samples 3019553 train steps 94049
Length 10
Episode 98516 total samples 3022057 train steps 94127
Length 18
Episode 98992 total samples 3024558 train steps 94205
Length 10
Episode 99468 total samples 3027062 train steps 94283
Length 8
Episode 99947 total samples 3029565 train steps 94361
Length 8
plot not available. Move on.
In [ ]:
Content source: febert/DeepRL
Similar notebooks: