In [1]:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
In [2]:
from dqn_learning import *
# import cPickle
# %load_ext autoreload
# %autoreload 2
%matplotlib inline
# import numpy as np
[2016-08-24 20:13:10,150] Site environment registry incorrect: Scoreboard did not register all envs: set(['AcrobotContinuous-v0'])
In [3]:
car = q_learning(gamma=1.,
lambda_=0.9,
# init_epsilon = 0.0,
# update_epsilon = False,
init_epsilon = 1.0,
end_epsilon = 0.1,
update_epsilon = True,
exploration_decrease_length = 1e6,
# N_0= 50,
environment = 'Acrobot-v0',
plot_resolution = 30,
nn_size_hidden = [400,300], # [100,100] --> 0.01, [100,100,100] --> similar [400,300]
nn_batch_size = 128,
nn_learning_rate = 2.5e-4,
replay_memory_size = 3e5, # larger than nn_batch_size*1e3 or doesn't train
descent_method = 'rmsprop',
# dropout_keep_prob = 0.5,
ema_decay_rate = 0.999,
train_frequency = 4.0,
from_pixels = True,
reg_weight = 0.0,
do_pretrain = False,
pretrain_steps = 0
)
[2016-08-24 20:13:12,067] Making new env: Acrobot-v0
conv 128 8 8 2
weights std = 0.125
[8, 8, 2, 32]
bias std = 0.0
conv 512 4 4 32
weights std = 0.0625
[4, 4, 32, 64]
bias std = 0.0
conv 576 3 3 64
weights std = 0.0589255650989
[3, 3, 64, 64]
bias std = 0.0
fc 7744
weights std = 0.0160706086633
bias std = 0.0
fc 512
weights std = 0.0441941738242
bias std = 0.0
Q-learning configured in nn graph
descending with rmsprop
using environment Acrobot-v0
qnn target q-learning False False
In [4]:
car.deepq_learning(num_iter = 100000, max_steps = 500000, max_learning_steps=50000)#, reset_replay_memory=True)
Episode 22 total samples 10525 train steps 16
Length 1287
Episode 28 total samples 13521 train steps 110
Length 2740
Episode 32 total samples 17019 train steps 219
Length 3314
Episode 37 total samples 19728 train steps 304
Length 671
Episode 45 total samples 22420 train steps 388
Length 477
Episode 50 total samples 25294 train steps 478
Length 1350
exploration 0.977235400001
/usr/lib/pymodules/python2.7/matplotlib/collections.py:548: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
if self._edgecolors == 'face':
Episode 56 total samples 28448 train steps 576
Length 1715
Episode 61 total samples 31532 train steps 673
Length 2196
Episode 68 total samples 34942 train steps 779
Length 2162
Episode 73 total samples 37537 train steps 861
Length 810
Episode 76 total samples 40293 train steps 947
Length 851
Episode 80 total samples 42934 train steps 1029
Length 449
Episode 85 total samples 45846 train steps 1120
Length 1584
Episode 90 total samples 48982 train steps 1218
Length 2348
exploration 0.954255700002
Episode 93 total samples 51791 train steps 1306
Length 1928
Episode 99 total samples 54687 train steps 1396
Length 848
Episode 104 total samples 57558 train steps 1486
Length 1582
Episode 110 total samples 60447 train steps 1576
Length 1084
Episode 115 total samples 62959 train steps 1655
Length 1331
Episode 120 total samples 65651 train steps 1739
Length 864
Episode 125 total samples 68582 train steps 1831
Length 1527
Episode 131 total samples 71922 train steps 1935
Length 1910
Episode 136 total samples 75185 train steps 2037
Length 1641
exploration 0.930449800002
Episode 138 total samples 78348 train steps 2136
Length 2140
Episode 142 total samples 81715 train steps 2241
Length 2560
Episode 146 total samples 84316 train steps 2322
Length 1495
Episode 150 total samples 87580 train steps 2424
Length 2017
Episode 156 total samples 90277 train steps 2509
Length 520
Episode 162 total samples 92934 train steps 2592
Length 347
Episode 168 total samples 95534 train steps 2673
Length 743
Episode 175 total samples 98272 train steps 2758
Length 513
Episode 180 total samples 100926 train steps 2841
Length 373
exploration 0.907742800003
Episode 184 total samples 103750 train steps 2930
Length 1015
Episode 192 total samples 106568 train steps 3018
Length 1575
Episode 197 total samples 109783 train steps 3118
Length 1787
Episode 203 total samples 112394 train steps 3200
Length 1947
Episode 208 total samples 114910 train steps 3278
Length 1340
Episode 214 total samples 117506 train steps 3360
Length 375
Episode 220 total samples 120634 train steps 3457
Length 1433
Episode 224 total samples 123818 train steps 3557
Length 1685
Episode 228 total samples 126494 train steps 3640
Length 1274
exploration 0.885184300004
Episode 234 total samples 129519 train steps 3735
Length 1170
Episode 239 total samples 132054 train steps 3814
Length 832
Episode 246 total samples 134886 train steps 3903
Length 980
Episode 252 total samples 137479 train steps 3984
Length 437
Episode 259 total samples 140096 train steps 4065
Length 799
Episode 264 total samples 142685 train steps 4146
Length 1039
Episode 270 total samples 145635 train steps 4239
Length 1646
Episode 274 total samples 148442 train steps 4326
Length 2532
Episode 280 total samples 151066 train steps 4408
Length 386
exploration 0.862546600005
Episode 287 total samples 153582 train steps 4487
Length 258
Episode 292 total samples 156448 train steps 4576
Length 911
Episode 299 total samples 159472 train steps 4671
Length 1507
Episode 306 total samples 163644 train steps 4801
Length 3961
Episode 309 total samples 166377 train steps 4887
Length 2128
Episode 315 total samples 169754 train steps 4992
Length 2658
Episode 323 total samples 173195 train steps 5100
Length 2232
Episode 331 total samples 176041 train steps 5189
Length 1040
Episode 337 total samples 178601 train steps 5269
Length 1972
exploration 0.839259100005
Episode 344 total samples 181144 train steps 5348
Length 1161
Episode 352 total samples 183945 train steps 5436
Length 1075
Episode 356 total samples 186846 train steps 5526
Length 1476
Episode 363 total samples 189423 train steps 5607
Length 989
Episode 369 total samples 192453 train steps 5702
Length 2265
Episode 376 total samples 195867 train steps 5808
Length 2337
Episode 383 total samples 198765 train steps 5899
Length 906
Episode 389 total samples 201482 train steps 5984
Length 707
Episode 395 total samples 204176 train steps 6068
Length 1287
exploration 0.816241600006
Episode 400 total samples 206684 train steps 6146
Length 1074
Episode 408 total samples 209215 train steps 6225
Length 224
Episode 413 total samples 211898 train steps 6309
Length 588
Episode 419 total samples 215110 train steps 6410
Length 1995
Episode 425 total samples 217661 train steps 6489
Length 784
Episode 430 total samples 220343 train steps 6573
Length 832
Episode 436 total samples 222950 train steps 6655
Length 955
Episode 442 total samples 225625 train steps 6738
Length 1091
Episode 450 total samples 228278 train steps 6821
Length 1008
exploration 0.792518500007
Episode 455 total samples 230870 train steps 6902
Length 669
Episode 463 total samples 233527 train steps 6985
Length 435
Episode 469 total samples 237240 train steps 7101
Length 3043
Episode 471 total samples 240112 train steps 7191
Length 4684
Episode 478 total samples 243001 train steps 7281
Length 870
Episode 486 total samples 245793 train steps 7369
Length 744
Episode 491 total samples 248697 train steps 7459
Length 1344
Episode 495 total samples 251841 train steps 7558
Length 2001
Episode 504 total samples 254471 train steps 7640
Length 656
exploration 0.769814200008
Episode 512 total samples 257413 train steps 7732
Length 1372
Episode 518 total samples 260099 train steps 7816
Length 1213
Episode 524 total samples 262794 train steps 7900
Length 607
Episode 530 total samples 265322 train steps 7979
Length 1181
Episode 536 total samples 268076 train steps 8065
Length 859
Episode 542 total samples 270708 train steps 8147
Length 2385
Episode 551 total samples 273575 train steps 8237
Length 834
Episode 557 total samples 276488 train steps 8328
Length 960
Episode 564 total samples 279499 train steps 8422
Length 1306
exploration 0.747234100008
Episode 572 total samples 282093 train steps 8503
Length 1118
Episode 579 total samples 285441 train steps 8608
Length 2333
Episode 586 total samples 288052 train steps 8689
Length 302
Episode 593 total samples 291770 train steps 8805
Length 2466
Episode 599 total samples 294329 train steps 8885
Length 929
Episode 606 total samples 297254 train steps 8977
Length 925
Episode 613 total samples 300491 train steps 9078
Length 2250
Episode 620 total samples 303083 train steps 9159
Length 689
Episode 627 total samples 306074 train steps 9252
Length 1035
exploration 0.724533400009
Episode 635 total samples 309170 train steps 9349
Length 1878
Episode 641 total samples 311966 train steps 9436
Length 1270
Episode 647 total samples 314561 train steps 9518
Length 1397
Episode 653 total samples 317504 train steps 9609
Length 1782
Episode 658 total samples 320119 train steps 9691
Length 1149
Episode 664 total samples 322833 train steps 9776
Length 1756
Episode 670 total samples 325426 train steps 9857
Length 395
Episode 678 total samples 327967 train steps 9936
Length 1293
Episode 685 total samples 330810 train steps 10025
Length 686
exploration 0.70195240001
Episode 692 total samples 333644 train steps 10114
Length 1524
Episode 698 total samples 336434 train steps 10201
Length 706
Episode 703 total samples 338950 train steps 10280
Length 817
Episode 709 total samples 341517 train steps 10360
Length 1410
Episode 716 total samples 344163 train steps 10443
Length 483
Episode 721 total samples 346700 train steps 10522
Length 964
Episode 727 total samples 349949 train steps 10623
Length 2024
Episode 733 total samples 352676 train steps 10709
Length 523
Episode 740 total samples 355412 train steps 10794
Length 584
exploration 0.679259800011
Episode 745 total samples 358624 train steps 10894
Length 1485
Episode 753 total samples 361427 train steps 10982
Length 762
Episode 760 total samples 363994 train steps 11062
Length 1072
Episode 769 total samples 366667 train steps 11146
Length 400
Episode 777 total samples 369434 train steps 11232
Length 773
Episode 783 total samples 372054 train steps 11314
Length 795
Episode 789 total samples 374886 train steps 11403
Length 948
Episode 796 total samples 377590 train steps 11487
Length 1105
Episode 802 total samples 380150 train steps 11567
Length 809
exploration 0.656735500011
Episode 808 total samples 382800 train steps 11650
Length 1010
Episode 817 total samples 385476 train steps 11734
Length 421
Episode 824 total samples 388267 train steps 11821
Length 1137
Episode 835 total samples 391066 train steps 11908
Length 676
Episode 843 total samples 394225 train steps 12007
Length 1320
Episode 848 total samples 397108 train steps 12097
Length 1523
Episode 853 total samples 399717 train steps 12179
Length 1825
Episode 860 total samples 402388 train steps 12262
Length 627
Episode 867 total samples 405211 train steps 12350
Length 1116
exploration 0.633845800012
Episode 873 total samples 407913 train steps 12435
Length 545
Episode 881 total samples 410543 train steps 12517
Length 692
Episode 885 total samples 413602 train steps 12613
Length 1815
Episode 890 total samples 416139 train steps 12692
Length 853
Episode 897 total samples 418830 train steps 12776
Length 971
Episode 904 total samples 421623 train steps 12863
Length 1588
Episode 910 total samples 424275 train steps 12946
Length 676
Episode 916 total samples 426867 train steps 13027
Length 706
Episode 920 total samples 429575 train steps 13112
Length 2168
exploration 0.611172100013
Episode 928 total samples 432519 train steps 13204
Length 976
Episode 935 total samples 435029 train steps 13282
Length 563
Episode 940 total samples 437583 train steps 13362
Length 760
Episode 945 total samples 440332 train steps 13448
Length 947
Episode 955 total samples 442959 train steps 13530
Length 489
Episode 964 total samples 445726 train steps 13616
Length 680
Episode 968 total samples 448393 train steps 13700
Length 1496
Episode 975 total samples 450927 train steps 13779
Length 537
Episode 982 total samples 453510 train steps 13860
Length 498
Episode 988 total samples 456196 train steps 13944
Length 474
exploration 0.588451600014
Episode 996 total samples 458705 train steps 14022
Length 376
Episode 1002 total samples 461432 train steps 14107
Length 555
Episode 1007 total samples 463934 train steps 14185
Length 726
Episode 1013 total samples 466545 train steps 14267
Length 1368
Episode 1022 total samples 469133 train steps 14348
Length 404
Episode 1028 total samples 471829 train steps 14432
Length 770
Episode 1035 total samples 474577 train steps 14518
Length 637
Episode 1040 total samples 477332 train steps 14604
Length 949
Episode 1048 total samples 479916 train steps 14685
Length 436
exploration 0.565872400014
Episode 1051 total samples 483089 train steps 14784
Length 1449
Episode 1059 total samples 485730 train steps 14867
Length 876
Episode 1065 total samples 488423 train steps 14951
Length 669
Episode 1074 total samples 491301 train steps 15041
Length 1470
Episode 1080 total samples 494093 train steps 15128
Length 750
Episode 1089 total samples 496931 train steps 15217
Length 1005
Episode 1096 total samples 499443 train steps 15295
Length 547
Episode 1105 total samples 502126 train steps 15379
Length 839
Episode 1110 total samples 504650 train steps 15458
Length 793
Episode 1117 total samples 507230 train steps 15538
Length 292
exploration 0.543324700015
Episode 1125 total samples 510077 train steps 15627
Length 1172
Episode 1131 total samples 513166 train steps 15724
Length 2999
Episode 1137 total samples 515739 train steps 15804
Length 1451
Episode 1145 total samples 518498 train steps 15891
Length 676
Episode 1153 total samples 521189 train steps 15975
Length 567
Episode 1159 total samples 523811 train steps 16057
Length 1137
Episode 1166 total samples 526346 train steps 16136
Length 714
Episode 1171 total samples 528977 train steps 16218
Length 1376
Episode 1178 total samples 531858 train steps 16308
Length 920
exploration 0.520813000016
Episode 1184 total samples 534561 train steps 16393
Length 662
Episode 1191 total samples 537153 train steps 16474
Length 214
Episode 1199 total samples 539906 train steps 16560
Length 561
Episode 1206 total samples 542552 train steps 16642
Length 749
Episode 1213 total samples 545673 train steps 16740
Length 2083
Episode 1222 total samples 548260 train steps 16821
Length 1183
Episode 1230 total samples 550928 train steps 16904
Length 500
Episode 1236 total samples 553635 train steps 16989
Length 1400
Episode 1245 total samples 556319 train steps 17072
Length 390
exploration 0.498233800016
Episode 1253 total samples 559369 train steps 17168
Length 1587
Episode 1259 total samples 562165 train steps 17255
Length 1731
Episode 1265 total samples 564854 train steps 17339
Length 581
Episode 1274 total samples 567879 train steps 17434
Length 1064
Episode 1282 total samples 570991 train steps 17531
Length 1403
Episode 1289 total samples 573582 train steps 17612
Length 577
Episode 1296 total samples 576236 train steps 17695
Length 724
Episode 1304 total samples 578862 train steps 17777
Length 708
Episode 1313 total samples 582061 train steps 17877
Length 1486
exploration 0.475685200016
Episode 1319 total samples 584885 train steps 17965
Length 916
Episode 1326 total samples 587425 train steps 18045
Length 620
Episode 1333 total samples 590019 train steps 18126
Length 702
Episode 1342 total samples 592688 train steps 18209
Length 1317
Episode 1348 total samples 595833 train steps 18307
Length 2147
Episode 1355 total samples 598563 train steps 18393
Length 819
Episode 1361 total samples 601228 train steps 18476
Length 1374
Episode 1368 total samples 604158 train steps 18567
Length 1345
Episode 1373 total samples 606933 train steps 18654
Length 889
exploration 0.452958400015
Episode 1380 total samples 609711 train steps 18741
Length 1692
Episode 1388 total samples 612324 train steps 18823
Length 411
Episode 1397 total samples 614878 train steps 18902
Length 611
Episode 1407 total samples 617771 train steps 18993
Length 917
Episode 1412 total samples 620316 train steps 19072
Length 1832
Episode 1419 total samples 623425 train steps 19170
Length 1838
Episode 1425 total samples 626143 train steps 19254
Length 462
Episode 1431 total samples 628801 train steps 19338
Length 588
Episode 1437 total samples 631411 train steps 19419
Length 1746
exploration 0.430118200014
Episode 1444 total samples 634469 train steps 19515
Length 1561
Episode 1448 total samples 637006 train steps 19594
Length 3012
Episode 1452 total samples 639623 train steps 19676
Length 2135
Episode 1460 total samples 642669 train steps 19771
Length 1091
Episode 1466 total samples 645324 train steps 19854
Length 442
Episode 1473 total samples 648079 train steps 19940
Length 757
Episode 1478 total samples 650629 train steps 20020
Length 742
Episode 1486 total samples 653397 train steps 20106
Length 619
Episode 1492 total samples 656075 train steps 20190
Length 636
exploration 0.407416600014
Episode 1500 total samples 658775 train steps 20274
Length 697
Episode 1505 total samples 661296 train steps 20353
Length 811
Episode 1511 total samples 664081 train steps 20440
Length 1249
Episode 1517 total samples 666876 train steps 20527
Length 1649
Episode 1523 total samples 669739 train steps 20617
Length 759
Episode 1534 total samples 672523 train steps 20704
Length 789
Episode 1539 total samples 675622 train steps 20801
Length 1675
Episode 1546 total samples 678312 train steps 20885
Length 616
Episode 1552 total samples 681015 train steps 20969
Length 1097
exploration 0.384865300013
Episode 1560 total samples 683968 train steps 21061
Length 970
Episode 1567 total samples 687213 train steps 21163
Length 1537
Episode 1570 total samples 689819 train steps 21244
Length 1118
Episode 1577 total samples 692369 train steps 21324
Length 1462
Episode 1583 total samples 695131 train steps 21410
Length 1393
Episode 1591 total samples 697780 train steps 21493
Length 501
Episode 1599 total samples 700395 train steps 21575
Length 366
Episode 1603 total samples 702973 train steps 21655
Length 1251
Episode 1609 total samples 705815 train steps 21744
Length 1697
Episode 1617 total samples 708430 train steps 21826
Length 291
exploration 0.362252800013
Episode 1623 total samples 711088 train steps 21909
Length 1156
Episode 1631 total samples 713864 train steps 21996
Length 959
Episode 1638 total samples 716437 train steps 22076
Length 816
Episode 1647 total samples 719545 train steps 22173
Length 1512
Episode 1654 total samples 722338 train steps 22261
Length 1484
Episode 1660 total samples 725188 train steps 22350
Length 1317
Episode 1667 total samples 728282 train steps 22446
Length 1994
Episode 1674 total samples 730894 train steps 22528
Length 581
Episode 1682 total samples 733540 train steps 22611
Length 995
exploration 0.339667300012
Episode 1688 total samples 736144 train steps 22692
Length 428
Episode 1696 total samples 739149 train steps 22786
Length 2114
Episode 1702 total samples 741872 train steps 22871
Length 816
Episode 1708 total samples 744633 train steps 22957
Length 634
Episode 1714 total samples 747227 train steps 23038
Length 496
Episode 1722 total samples 750468 train steps 23140
Length 1755
Episode 1727 total samples 753086 train steps 23221
Length 694
Episode 1735 total samples 755850 train steps 23308
Length 1008
Episode 1739 total samples 759131 train steps 23410
Length 2395
exploration 0.316782100011
Episode 1744 total samples 761717 train steps 23491
Length 480
Episode 1750 total samples 764263 train steps 23571
Length 346
Episode 1755 total samples 766898 train steps 23653
Length 884
Episode 1760 total samples 769976 train steps 23749
Length 1162
Episode 1762 total samples 772498 train steps 23828
Length 1907
Episode 1769 total samples 775240 train steps 23914
Length 824
Episode 1775 total samples 777741 train steps 23992
Length 300
Episode 1781 total samples 780254 train steps 24070
Length 705
Episode 1788 total samples 783075 train steps 24159
Length 798
exploration 0.294122800011
Episode 1795 total samples 786050 train steps 24252
Length 1548
Episode 1802 total samples 789145 train steps 24348
Length 1509
Episode 1807 total samples 791790 train steps 24431
Length 1731
Episode 1814 total samples 794629 train steps 24520
Length 787
Episode 1820 total samples 797324 train steps 24604
Length 928
Episode 1824 total samples 799957 train steps 24686
Length 1433
Episode 1829 total samples 802513 train steps 24766
Length 1719
Episode 1835 total samples 805312 train steps 24853
Length 1889
Episode 1840 total samples 807871 train steps 24933
Length 1368
exploration 0.27140950001
Episode 1846 total samples 811518 train steps 25047
Length 2868
Episode 1851 total samples 814421 train steps 25138
Length 1003
Episode 1857 total samples 817147 train steps 25223
Length 688
Episode 1861 total samples 820229 train steps 25320
Length 2158
Episode 1867 total samples 822790 train steps 25400
Length 336
Episode 1875 total samples 826452 train steps 25514
Length 2501
Episode 1878 total samples 829311 train steps 25603
Length 3393
Episode 1885 total samples 832035 train steps 25689
Length 642
Episode 1894 total samples 834740 train steps 25773
Length 417
exploration 0.248734000009
Episode 1900 total samples 837281 train steps 25853
Length 570
Episode 1904 total samples 840024 train steps 25938
Length 1846
Episode 1909 total samples 842547 train steps 26017
Length 809
Episode 1915 total samples 845251 train steps 26102
Length 455
Episode 1921 total samples 847947 train steps 26186
Length 1339
Episode 1928 total samples 850668 train steps 26271
Length 1320
Episode 1933 total samples 853199 train steps 26350
Length 1836
Episode 1941 total samples 856484 train steps 26453
Length 1581
Episode 1949 total samples 859247 train steps 26539
Length 667
exploration 0.226113400009
Episode 1953 total samples 861956 train steps 26624
Length 1706
Episode 1959 total samples 864514 train steps 26704
Length 991
Episode 1965 total samples 867490 train steps 26797
Length 2074
Episode 1971 total samples 870111 train steps 26878
Length 948
Episode 1976 total samples 872892 train steps 26965
Length 1528
Episode 1983 total samples 875429 train steps 27045
Length 343
Episode 1987 total samples 878102 train steps 27128
Length 1717
Episode 1992 total samples 880921 train steps 27216
Length 764
Episode 1997 total samples 883811 train steps 27307
Length 1010
exploration 0.203463100009
Episode 2003 total samples 886662 train steps 27396
Length 2054
Episode 2008 total samples 889187 train steps 27475
Length 1047
Episode 2017 total samples 891870 train steps 27558
Length 530
Episode 2023 total samples 894553 train steps 27642
Length 556
Episode 2029 total samples 897372 train steps 27730
Length 879
Episode 2035 total samples 901111 train steps 27847
Length 2656
Episode 2041 total samples 904897 train steps 27966
Length 2817
Episode 2047 total samples 907585 train steps 28050
Length 1786
Episode 2050 total samples 910320 train steps 28135
Length 2444
exploration 0.180712000009
Episode 2055 total samples 912842 train steps 28214
Length 1777
Episode 2060 total samples 915364 train steps 28293
Length 488
Episode 2065 total samples 918227 train steps 28382
Length 1176
Episode 2071 total samples 921372 train steps 28480
Length 1329
Episode 2075 total samples 924086 train steps 28565
Length 2225
Episode 2080 total samples 926608 train steps 28644
Length 926
Episode 2084 total samples 929331 train steps 28729
Length 686
Episode 2088 total samples 932028 train steps 28813
Length 3323
Episode 2092 total samples 934700 train steps 28897
Length 687
exploration 0.157621600009
Episode 2097 total samples 937884 train steps 28996
Length 1447
Episode 2101 total samples 941031 train steps 29095
Length 2908
Episode 2106 total samples 943601 train steps 29175
Length 803
Episode 2111 total samples 946549 train steps 29267
Length 2262
Episode 2116 total samples 949200 train steps 29350
Length 1184
Episode 2121 total samples 952659 train steps 29458
Length 2195
Episode 2126 total samples 955402 train steps 29544
Length 1378
Episode 2130 total samples 958847 train steps 29651
Length 2162
exploration 0.13497760001
Episode 2136 total samples 961946 train steps 29748
Length 1619
Episode 2140 total samples 965321 train steps 29854
Length 2285
Episode 2145 total samples 968057 train steps 29939
Length 982
Episode 2149 total samples 970671 train steps 30021
Length 1663
Episode 2153 total samples 974455 train steps 30139
Length 2867
Episode 2157 total samples 978166 train steps 30255
Length 3055
Episode 2162 total samples 980768 train steps 30336
Length 1333
Episode 2166 total samples 983343 train steps 30417
Length 1438
Episode 2170 total samples 986346 train steps 30511
Length 2447
exploration 0.11228860001
Episode 2172 total samples 988915 train steps 30591
Length 657
Episode 2175 total samples 991834 train steps 30682
Length 1955
Episode 2179 total samples 995657 train steps 30802
Length 2928
Episode 2182 total samples 998830 train steps 30901
Length 2651
Episode 2185 total samples 1002158 train steps 31005
Length 2334
Episode 2188 total samples 1005435 train steps 31107
Length 1635
Episode 2192 total samples 1007955 train steps 31186
Length 644
Episode 2194 total samples 1013685 train steps 31365
Length 9605
exploration 0.0999991000096
Episode 2197 total samples 1016356 train steps 31449
Length 960
Episode 2201 total samples 1019353 train steps 31542
Length 1505
Episode 2204 total samples 1022633 train steps 31645
Length 1938
Episode 2207 total samples 1025158 train steps 31724
Length 2535
Episode 2210 total samples 1028005 train steps 31813
Length 1980
Episode 2213 total samples 1031214 train steps 31913
Length 2359
Episode 2217 total samples 1034261 train steps 32008
Length 1784
Episode 2220 total samples 1037008 train steps 32094
Length 2967
Episode 2223 total samples 1039632 train steps 32176
Length 1981
exploration 0.0999991000096
Episode 2228 total samples 1042331 train steps 32260
Length 738
Episode 2231 total samples 1045075 train steps 32346
Length 1088
Episode 2235 total samples 1047700 train steps 32428
Length 1261
Episode 2238 total samples 1051178 train steps 32537
Length 2356
Episode 2240 total samples 1055540 train steps 32673
Length 4150
Episode 2242 total samples 1058073 train steps 32752
Length 2584
Episode 2245 total samples 1061042 train steps 32845
Length 4035
Episode 2249 total samples 1064584 train steps 32956
Length 2511
exploration 0.0999991000096
Episode 2253 total samples 1067134 train steps 33035
Length 1321
Episode 2258 total samples 1070022 train steps 33126
Length 1104
Episode 2262 total samples 1072912 train steps 33216
Length 2555
Episode 2266 total samples 1075594 train steps 33300
Length 1387
Episode 2268 total samples 1078506 train steps 33391
Length 2316
Episode 2271 total samples 1081111 train steps 33472
Length 1742
Episode 2275 total samples 1084394 train steps 33575
Length 1711
Episode 2279 total samples 1087416 train steps 33669
Length 1853
Episode 2283 total samples 1090065 train steps 33752
Length 2148
exploration 0.0999991000096
Episode 2286 total samples 1093192 train steps 33850
Length 1465
Episode 2288 total samples 1096167 train steps 33943
Length 2942
Episode 2293 total samples 1100247 train steps 34070
Length 3289
Episode 2297 total samples 1102953 train steps 34155
Length 2053
Episode 2301 total samples 1106120 train steps 34254
Length 1440
Episode 2305 total samples 1109602 train steps 34363
Length 1980
Episode 2308 total samples 1113244 train steps 34476
Length 2956
Episode 2313 total samples 1115854 train steps 34558
Length 1159
exploration 0.0999991000096
Episode 2316 total samples 1118866 train steps 34652
Length 1876
Episode 2320 total samples 1121987 train steps 34750
Length 2216
Episode 2323 total samples 1125279 train steps 34852
Length 2546
Episode 2326 total samples 1128011 train steps 34938
Length 840
Episode 2331 total samples 1131437 train steps 35045
Length 2385
Episode 2334 total samples 1134392 train steps 35137
Length 2454
Episode 2337 total samples 1137031 train steps 35220
Length 1143
Episode 2343 total samples 1139724 train steps 35304
Length 1192
Episode 2346 total samples 1143293 train steps 35415
Length 2307
exploration 0.0999991000096
Episode 2351 total samples 1145914 train steps 35497
Length 1489
Episode 2355 total samples 1148547 train steps 35580
Length 1146
Episode 2359 total samples 1151140 train steps 35661
Length 1159
Episode 2363 total samples 1155074 train steps 35784
Length 2883
Episode 2367 total samples 1157653 train steps 35864
Length 1825
Episode 2371 total samples 1160484 train steps 35953
Length 882
Episode 2374 total samples 1163135 train steps 36035
Length 1634
Episode 2378 total samples 1165947 train steps 36123
Length 758
Episode 2381 total samples 1170705 train steps 36272
Length 7446
exploration 0.0999991000096
Episode 2385 total samples 1173265 train steps 36352
Length 2503
Episode 2387 total samples 1176077 train steps 36440
Length 2775
Episode 2392 total samples 1179177 train steps 36537
Length 1954
Episode 2396 total samples 1182551 train steps 36642
Length 2372
Episode 2400 total samples 1185064 train steps 36721
Length 2352
Episode 2403 total samples 1187808 train steps 36806
Length 2376
Episode 2407 total samples 1190652 train steps 36895
Length 2491
Episode 2412 total samples 1193639 train steps 36989
Length 1894
exploration 0.0999991000096
Episode 2416 total samples 1196936 train steps 37092
Length 1972
Episode 2419 total samples 1200273 train steps 37196
Length 2123
Episode 2422 total samples 1203143 train steps 37286
Length 1235
Episode 2427 total samples 1207013 train steps 37407
Length 2945
Episode 2432 total samples 1210049 train steps 37502
Length 1830
Episode 2437 total samples 1214319 train steps 37635
Length 4235
Episode 2441 total samples 1217223 train steps 37726
Length 2533
Episode 2445 total samples 1221390 train steps 37856
Length 4893
exploration 0.0999991000096
Episode 2449 total samples 1224310 train steps 37947
Length 1406
Episode 2452 total samples 1227121 train steps 38035
Length 1449
Episode 2455 total samples 1229901 train steps 38122
Length 1031
Episode 2460 total samples 1232955 train steps 38217
Length 1210
Episode 2463 total samples 1235520 train steps 38297
Length 1025
Episode 2467 total samples 1239101 train steps 38409
Length 3009
Episode 2471 total samples 1242224 train steps 38507
Length 2536
Episode 2475 total samples 1247505 train steps 38672
Length 6001
exploration 0.0999991000096
Episode 2478 total samples 1250187 train steps 38756
Length 1645
Episode 2480 total samples 1253004 train steps 38844
Length 1296
Episode 2483 total samples 1255620 train steps 38926
Length 635
Episode 2488 total samples 1258222 train steps 39007
Length 1553
Episode 2490 total samples 1260990 train steps 39093
Length 2878
Episode 2494 total samples 1263655 train steps 39177
Length 1812
Episode 2499 total samples 1266662 train steps 39271
Length 1902
Episode 2503 total samples 1269355 train steps 39355
Length 1826
Episode 2507 total samples 1272733 train steps 39460
Length 2281
exploration 0.0999991000096
Episode 2511 total samples 1275321 train steps 39541
Length 1257
Episode 2514 total samples 1277845 train steps 39620
Length 1524
Episode 2518 total samples 1281377 train steps 39731
Length 3408
Episode 2521 total samples 1285075 train steps 39846
Length 2625
Episode 2525 total samples 1288304 train steps 39947
Length 1497
Episode 2529 total samples 1291289 train steps 40040
Length 2758
Episode 2533 total samples 1295489 train steps 40172
Length 4123
Episode 2536 total samples 1298376 train steps 40262
Length 2162
exploration 0.0999991000096
Episode 2538 total samples 1301017 train steps 40344
Length 2474
Episode 2540 total samples 1304302 train steps 40447
Length 4538
Episode 2544 total samples 1307629 train steps 40551
Length 2459
Episode 2548 total samples 1310372 train steps 40637
Length 1960
Episode 2551 total samples 1312980 train steps 40718
Length 2125
Episode 2554 total samples 1315804 train steps 40806
Length 3330
Episode 2560 total samples 1318446 train steps 40889
Length 1159
Episode 2564 total samples 1321095 train steps 40972
Length 1963
Episode 2567 total samples 1324160 train steps 41067
Length 2105
exploration 0.0999991000096
Episode 2571 total samples 1326915 train steps 41154
Length 1931
Episode 2575 total samples 1329864 train steps 41246
Length 2495
Episode 2579 total samples 1332871 train steps 41340
Length 1989
Episode 2583 total samples 1335749 train steps 41430
Length 3051
Episode 2586 total samples 1338330 train steps 41510
Length 1468
Episode 2591 total samples 1341834 train steps 41620
Length 2697
Episode 2596 total samples 1345869 train steps 41746
Length 3943
Episode 2599 total samples 1349104 train steps 41847
Length 1510
exploration 0.0999991000096
Episode 2601 total samples 1352347 train steps 41948
Length 2818
Episode 2606 total samples 1355414 train steps 42044
Length 1155
Episode 2608 total samples 1357946 train steps 42123
Length 3602
Episode 2612 total samples 1361139 train steps 42223
Length 1791
Episode 2616 total samples 1363959 train steps 42311
Length 965
Episode 2619 total samples 1366544 train steps 42392
Length 1937
Episode 2624 total samples 1369291 train steps 42478
Length 1029
Episode 2627 total samples 1371906 train steps 42560
Length 1462
Episode 2630 total samples 1374687 train steps 42646
Length 1587
Episode 2633 total samples 1377219 train steps 42726
Length 3085
exploration 0.0999991000096
Episode 2636 total samples 1379743 train steps 42804
Length 637
Episode 2640 total samples 1383526 train steps 42923
Length 2766
Episode 2643 total samples 1386251 train steps 43008
Length 2226
Episode 2647 total samples 1389062 train steps 43096
Length 1733
Episode 2651 total samples 1391661 train steps 43177
Length 2001
Episode 2656 total samples 1394181 train steps 43256
Length 1242
Episode 2660 total samples 1397330 train steps 43354
Length 2141
Episode 2664 total samples 1399832 train steps 43432
Length 1151
Episode 2669 total samples 1402895 train steps 43528
Length 1475
exploration 0.0999991000096
Episode 2673 total samples 1406144 train steps 43629
Length 1941
Episode 2676 total samples 1408682 train steps 43709
Length 2041
Episode 2679 total samples 1411948 train steps 43811
Length 4416
Episode 2681 total samples 1414547 train steps 43892
Length 1639
Episode 2686 total samples 1417145 train steps 43973
Length 1775
Episode 2691 total samples 1420999 train steps 44094
Length 3037
Episode 2694 total samples 1423623 train steps 44176
Length 547
Episode 2697 total samples 1426505 train steps 44266
Length 1638
exploration 0.0999991000096
Episode 2700 total samples 1429416 train steps 44357
Length 1766
Episode 2703 total samples 1432331 train steps 44448
Length 1223
Episode 2708 total samples 1435334 train steps 44542
Length 1555
Episode 2712 total samples 1438457 train steps 44639
Length 2077
Episode 2715 total samples 1442147 train steps 44755
Length 3233
Episode 2719 total samples 1444813 train steps 44838
Length 1380
Episode 2722 total samples 1447636 train steps 44926
Length 1550
Episode 2726 total samples 1451331 train steps 45042
Length 3224
exploration 0.0999991000096
Episode 2731 total samples 1454213 train steps 45132
Length 1326
Episode 2733 total samples 1457334 train steps 45229
Length 3201
Episode 2736 total samples 1460938 train steps 45342
Length 2973
Episode 2739 total samples 1463442 train steps 45420
Length 1112
Episode 2743 total samples 1466449 train steps 45514
Length 2545
Episode 2745 total samples 1469895 train steps 45622
Length 2166
Episode 2747 total samples 1472472 train steps 45702
Length 3919
Episode 2750 total samples 1475424 train steps 45794
Length 2365
Episode 2754 total samples 1478919 train steps 45904
Length 2881
exploration 0.0999991000096
Episode 2758 total samples 1481625 train steps 45988
Length 2143
Episode 2762 total samples 1485304 train steps 46103
Length 3502
Episode 2766 total samples 1488424 train steps 46201
Length 1325
Episode 2770 total samples 1491400 train steps 46294
Length 1635
Episode 2774 total samples 1494674 train steps 46396
Length 2361
Episode 2778 total samples 1497242 train steps 46476
Length 1833
Episode 2781 total samples 1499961 train steps 46561
Length 1140
Episode 2786 total samples 1503084 train steps 46659
Length 1332
exploration 0.0999991000096
Episode 2789 total samples 1507245 train steps 46789
Length 3519
Episode 2792 total samples 1510696 train steps 46897
Length 2385
Episode 2796 total samples 1513392 train steps 46981
Length 2773
Episode 2800 total samples 1516344 train steps 47073
Length 2022
Episode 2803 total samples 1519459 train steps 47171
Length 3122
Episode 2805 total samples 1522324 train steps 47260
Length 1405
Episode 2808 total samples 1525632 train steps 47363
Length 2308
Episode 2811 total samples 1528418 train steps 47451
Length 1373
exploration 0.0999991000096
Episode 2814 total samples 1531709 train steps 47553
Length 3742
Episode 2817 total samples 1535071 train steps 47658
Length 3467
Episode 2819 total samples 1537759 train steps 47742
Length 3379
Episode 2823 total samples 1540269 train steps 47821
Length 1158
Episode 2829 total samples 1543602 train steps 47925
Length 2294
Episode 2834 total samples 1546742 train steps 48023
Length 1377
Episode 2839 total samples 1549491 train steps 48109
Length 1309
Episode 2840 total samples 1552301 train steps 48197
Length 5619
Episode 2844 total samples 1555374 train steps 48293
Length 1793
exploration 0.0999991000096
Episode 2848 total samples 1558989 train steps 48406
Length 2355
Episode 2851 total samples 1561668 train steps 48490
Length 1301
Episode 2855 total samples 1565053 train steps 48595
Length 2279
Episode 2859 total samples 1567682 train steps 48678
Length 1003
Episode 2863 total samples 1571164 train steps 48786
Length 2625
Episode 2866 total samples 1574374 train steps 48887
Length 4390
Episode 2869 total samples 1577527 train steps 48985
Length 1620
Episode 2873 total samples 1580114 train steps 49066
Length 1491
exploration 0.0999991000096
Episode 2875 total samples 1584395 train steps 49200
Length 5290
Episode 2879 total samples 1587780 train steps 49306
Length 2940
Episode 2884 total samples 1590625 train steps 49395
Length 762
Episode 2890 total samples 1593564 train steps 49486
Length 1638
Episode 2894 total samples 1597401 train steps 49606
Length 3586
Episode 2898 total samples 1600479 train steps 49702
Length 2086
Episode 2902 total samples 1603395 train steps 49794
Length 2004
Episode 2906 total samples 1606385 train steps 49887
Length 2844
exploration 0.0999991000096
Episode 2909 total samples 1609281 train steps 49978
Length 2273
done training, I'm tired. I started by 0
plot not available. Move on.
In [ ]:
car.deepq_learning(num_iter = 100000, max_steps = 500000, max_learning_steps=50000)#, reset_replay_memory=True)
Episode 3 total samples 1614644 train steps 50145
Length 5108
Episode 8 total samples 1617449 train steps 50233
Length 1198
Episode 12 total samples 1620018 train steps 50313
Length 1809
Episode 14 total samples 1623014 train steps 50407
Length 3667
Episode 19 total samples 1626846 train steps 50526
Length 2973
Episode 23 total samples 1630091 train steps 50628
Length 1765
Episode 26 total samples 1632728 train steps 50710
Length 2567
Episode 29 total samples 1635554 train steps 50799
Length 2430
exploration 0.0999991000096
Episode 33 total samples 1638436 train steps 50889
Length 2167
Episode 38 total samples 1642302 train steps 51009
Length 2873
Episode 42 total samples 1644855 train steps 51089
Length 1671
Episode 45 total samples 1647668 train steps 51177
Length 1262
Episode 48 total samples 1650193 train steps 51256
Length 2428
Episode 52 total samples 1652984 train steps 51343
Length 2546
Episode 56 total samples 1655710 train steps 51428
Length 1509
Episode 58 total samples 1658334 train steps 51510
Length 1736
exploration 0.0999991000096
Episode 62 total samples 1661106 train steps 51597
Length 944
Episode 68 total samples 1664603 train steps 51706
Length 2806
Episode 72 total samples 1667718 train steps 51804
Length 2010
Episode 75 total samples 1670721 train steps 51898
Length 3253
Episode 77 total samples 1673452 train steps 51983
Length 3810
Episode 81 total samples 1676494 train steps 52078
Length 1839
Episode 86 total samples 1679468 train steps 52171
Length 1335
Episode 91 total samples 1682341 train steps 52261
Length 1569
Episode 94 total samples 1686466 train steps 52390
Length 3566
exploration 0.0999991000096
Episode 98 total samples 1689078 train steps 52471
Length 1064
Episode 101 total samples 1691596 train steps 52550
Length 2075
Episode 104 total samples 1694488 train steps 52640
Length 2890
Episode 106 total samples 1697716 train steps 52741
Length 2545
Episode 110 total samples 1700379 train steps 52824
Length 2058
Episode 113 total samples 1703488 train steps 52921
Length 3195
Episode 117 total samples 1707072 train steps 53033
Length 3838
Episode 122 total samples 1710435 train steps 53139
Length 1727
exploration 0.0999991000096
Episode 126 total samples 1714028 train steps 53251
Length 2189
Episode 130 total samples 1717160 train steps 53349
Length 2917
Episode 134 total samples 1719857 train steps 53433
Length 1445
Episode 137 total samples 1723042 train steps 53533
Length 1387
Episode 142 total samples 1726159 train steps 53630
Length 2559
Episode 147 total samples 1728781 train steps 53712
Length 650
Episode 150 total samples 1731542 train steps 53798
Length 1839
Episode 152 total samples 1734196 train steps 53881
Length 4191
Episode 154 total samples 1737831 train steps 53995
Length 2870
exploration 0.0999991000096
Episode 157 total samples 1740573 train steps 54080
Length 2656
Episode 160 total samples 1743354 train steps 54167
Length 2762
Episode 164 total samples 1746419 train steps 54263
Length 1397
Episode 168 total samples 1749633 train steps 54364
Length 1633
Episode 172 total samples 1753313 train steps 54479
Length 2560
Episode 176 total samples 1756095 train steps 54565
Length 1948
Episode 181 total samples 1758680 train steps 54646
Length 384
Episode 184 total samples 1761525 train steps 54735
Length 3656
exploration 0.0999991000096
Episode 187 total samples 1764298 train steps 54822
Length 2256
Episode 190 total samples 1767247 train steps 54914
Length 1361
Episode 195 total samples 1770532 train steps 55017
Length 2135
Episode 198 total samples 1773059 train steps 55096
Length 1265
Episode 201 total samples 1776007 train steps 55188
Length 3178
Episode 207 total samples 1778999 train steps 55281
Length 1147
Episode 211 total samples 1782026 train steps 55376
Length 1832
Episode 215 total samples 1784679 train steps 55459
Length 2032
Episode 221 total samples 1788556 train steps 55580
Length 3255
exploration 0.0999991000096
Episode 224 total samples 1791314 train steps 55666
Length 620
Episode 229 total samples 1794212 train steps 55757
Length 987
Episode 234 total samples 1796812 train steps 55838
Length 601
Episode 240 total samples 1799317 train steps 55916
Length 771
Episode 243 total samples 1804220 train steps 56069
Length 7405
Episode 246 total samples 1806898 train steps 56153
Length 3019
Episode 250 total samples 1810706 train steps 56272
Length 2652
Episode 254 total samples 1813873 train steps 56371
Length 1486
exploration 0.0999991000096
Episode 257 total samples 1816675 train steps 56459
Length 1723
Episode 259 total samples 1819457 train steps 56546
Length 564
Episode 261 total samples 1822084 train steps 56628
Length 2735
Episode 265 total samples 1824979 train steps 56718
Length 1262
Episode 269 total samples 1827534 train steps 56798
Length 1058
Episode 272 total samples 1830236 train steps 56882
Length 2270
Episode 276 total samples 1833049 train steps 56970
Length 3217
Episode 278 total samples 1836709 train steps 57085
Length 3585
Episode 283 total samples 1839270 train steps 57165
Length 1275
exploration 0.0999991000096
Episode 288 total samples 1841936 train steps 57248
Length 424
Episode 290 total samples 1844642 train steps 57333
Length 2428
Episode 293 total samples 1847566 train steps 57424
Length 1725
Episode 296 total samples 1850515 train steps 57516
Length 1299
Episode 301 total samples 1853399 train steps 57606
Length 1150
Episode 303 total samples 1856914 train steps 57716
Length 3663
Episode 308 total samples 1859770 train steps 57805
Length 888
Episode 312 total samples 1862535 train steps 57892
Length 584
exploration 0.0999991000096
Episode 316 total samples 1865248 train steps 57976
Length 844
Episode 322 total samples 1868284 train steps 58071
Length 1204
Episode 326 total samples 1872725 train steps 58210
Length 5474
Episode 332 total samples 1875512 train steps 58297
Length 1110
Episode 335 total samples 1878707 train steps 58397
Length 1966
Episode 339 total samples 1881676 train steps 58490
Length 2034
Episode 342 total samples 1884203 train steps 58569
Length 2199
Episode 345 total samples 1887438 train steps 58670
Length 3278
Episode 349 total samples 1890078 train steps 58752
Length 1361
exploration 0.0999991000096
Episode 354 total samples 1892681 train steps 58834
Length 946
Episode 358 total samples 1896712 train steps 58960
Length 3415
Episode 363 total samples 1899258 train steps 59039
Length 952
Episode 367 total samples 1904836 train steps 59214
Length 7094
Episode 371 total samples 1908111 train steps 59316
Length 2023
Episode 376 total samples 1911135 train steps 59410
Length 1173
Episode 380 total samples 1914303 train steps 59509
Length 1429
exploration 0.0999991000096
Episode 384 total samples 1916879 train steps 59590
Length 1247
Episode 388 total samples 1919662 train steps 59677
Length 1530
Episode 392 total samples 1922423 train steps 59763
Length 946
Episode 396 total samples 1925445 train steps 59858
Length 1232
Episode 400 total samples 1928784 train steps 59962
Length 1865
Episode 404 total samples 1931907 train steps 60060
Length 1723
Episode 408 total samples 1934878 train steps 60152
Length 1228
Episode 412 total samples 1938178 train steps 60256
Length 1659
Episode 415 total samples 1941817 train steps 60369
Length 2919
exploration 0.0999991000096
Episode 418 total samples 1944412 train steps 60450
Length 2434
Episode 422 total samples 1947337 train steps 60542
Length 2137
Episode 427 total samples 1950342 train steps 60636
Length 1244
Episode 432 total samples 1953210 train steps 60725
Length 819
Episode 435 total samples 1956183 train steps 60818
Length 1741
Episode 437 total samples 1959133 train steps 60910
Length 5541
Episode 440 total samples 1962477 train steps 61015
Length 4186
Episode 445 total samples 1966447 train steps 61139
Length 3556
exploration 0.0999991000096
Episode 448 total samples 1969144 train steps 61223
Length 3526
Episode 450 total samples 1971754 train steps 61305
Length 4575
Episode 453 total samples 1974674 train steps 61396
Length 1449
Episode 456 total samples 1977450 train steps 61483
Length 1004
Episode 459 total samples 1981528 train steps 61610
Length 3567
Episode 462 total samples 1984243 train steps 61695
Length 1449
Episode 467 total samples 1987044 train steps 61783
Length 2043
Episode 471 total samples 1990045 train steps 61876
Length 2024
Episode 474 total samples 1993163 train steps 61974
Length 3397
exploration 0.0999991000096
Episode 478 total samples 1996428 train steps 62076
Length 2265
Episode 483 total samples 2001915 train steps 62247
Length 6746
Episode 487 total samples 2005800 train steps 62369
Length 2843
Episode 491 total samples 2009005 train steps 62469
Length 1456
Episode 496 total samples 2011866 train steps 62558
Length 945
Episode 500 total samples 2015068 train steps 62658
Length 2435
Episode 503 total samples 2017951 train steps 62748
Length 2329
exploration 0.0999991000096
Episode 506 total samples 2022270 train steps 62883
Length 4962
Episode 509 total samples 2025661 train steps 62989
Length 2107
Episode 513 total samples 2028258 train steps 63071
Length 1413
Episode 516 total samples 2031148 train steps 63161
Length 1509
Episode 520 total samples 2033807 train steps 63244
Length 1588
Episode 522 total samples 2036381 train steps 63324
Length 2236
Episode 525 total samples 2039410 train steps 63419
Length 3719
Episode 529 total samples 2042309 train steps 63510
Length 1661
exploration 0.0999991000096
Episode 533 total samples 2045857 train steps 63621
Length 4008
Episode 538 total samples 2049213 train steps 63725
Length 1749
Episode 543 total samples 2052063 train steps 63814
Length 1020
Episode 546 total samples 2055169 train steps 63912
Length 1681
Episode 551 total samples 2058110 train steps 64003
Length 1240
Episode 554 total samples 2061117 train steps 64097
Length 1891
Episode 558 total samples 2064477 train steps 64202
Length 2466
Episode 562 total samples 2067675 train steps 64302
Length 2026
exploration 0.0999991000096
Episode 566 total samples 2070643 train steps 64395
Length 1618
Episode 571 total samples 2073540 train steps 64486
Length 2152
Episode 574 total samples 2076776 train steps 64587
Length 2605
Episode 576 total samples 2079359 train steps 64667
Length 1454
Episode 582 total samples 2082039 train steps 64751
Length 614
Episode 585 total samples 2086572 train steps 64893
Length 5934
Episode 589 total samples 2089763 train steps 64993
Length 1443
Episode 593 total samples 2092284 train steps 65071
Length 1468
exploration 0.0999991000096
Episode 597 total samples 2095221 train steps 65163
Length 1173
Episode 602 total samples 2098031 train steps 65251
Length 647
Episode 607 total samples 2100815 train steps 65338
Length 704
Episode 612 total samples 2103484 train steps 65421
Length 892
Episode 615 total samples 2107974 train steps 65562
Length 4075
Episode 619 total samples 2110987 train steps 65656
Length 2148
Episode 622 total samples 2114054 train steps 65752
Length 2613
Episode 626 total samples 2117738 train steps 65867
Length 3653
exploration 0.0999991000096
Episode 630 total samples 2120637 train steps 65957
Length 852
Episode 634 total samples 2123668 train steps 66052
Length 2168
Episode 638 total samples 2127558 train steps 66174
Length 3604
Episode 641 total samples 2130085 train steps 66253
Length 1680
Episode 643 total samples 2132958 train steps 66342
Length 3213
Episode 647 total samples 2135504 train steps 66422
Length 1705
Episode 651 total samples 2138034 train steps 66501
Length 1338
Episode 654 total samples 2141176 train steps 66599
Length 1944
Episode 658 total samples 2143770 train steps 66680
Length 368
exploration 0.0999991000096
Episode 663 total samples 2147070 train steps 66783
Length 1646
Episode 667 total samples 2149769 train steps 66868
Length 3375
Episode 674 total samples 2153320 train steps 66979
Length 2195
Episode 678 total samples 2156421 train steps 67076
Length 1892
Episode 682 total samples 2159351 train steps 67167
Length 1342
Episode 686 total samples 2161968 train steps 67249
Length 886
Episode 691 total samples 2164938 train steps 67342
Length 1140
Episode 694 total samples 2167585 train steps 67425
Length 2558
Episode 696 total samples 2170196 train steps 67506
Length 1134
exploration 0.0999991000096
Episode 699 total samples 2172953 train steps 67592
Length 1630
Episode 703 total samples 2175578 train steps 67674
Length 1284
Episode 705 total samples 2178164 train steps 67755
Length 3033
Episode 708 total samples 2180923 train steps 67841
Length 2848
Episode 712 total samples 2184149 train steps 67942
Length 1577
Episode 714 total samples 2187114 train steps 68035
Length 3406
Episode 717 total samples 2190681 train steps 68146
Length 3654
Episode 720 total samples 2194008 train steps 68250
Length 3283
exploration 0.0999991000096
Episode 726 total samples 2197100 train steps 68347
Length 1190
Episode 730 total samples 2200299 train steps 68447
Length 2832
Episode 735 total samples 2202965 train steps 68530
Length 786
Episode 739 total samples 2205943 train steps 68623
Length 1431
Episode 742 total samples 2208660 train steps 68708
Length 1839
Episode 746 total samples 2211619 train steps 68801
Length 1574
Episode 749 total samples 2215851 train steps 68933
Length 4878
Episode 754 total samples 2218676 train steps 69021
Length 911
Episode 759 total samples 2221838 train steps 69120
Length 2101
exploration 0.0999991000096
Episode 761 total samples 2224447 train steps 69201
Length 3518
Episode 763 total samples 2227437 train steps 69295
Length 3409
Episode 766 total samples 2230559 train steps 69392
Length 2378
Episode 770 total samples 2233266 train steps 69477
Length 1998
Episode 773 total samples 2235915 train steps 69560
Length 1878
Episode 777 total samples 2238913 train steps 69654
Length 1330
Episode 782 total samples 2242411 train steps 69763
Length 2499
Episode 785 total samples 2245660 train steps 69864
Length 2021
exploration 0.0999991000096
Episode 790 total samples 2248491 train steps 69953
Length 1590
Episode 795 total samples 2252748 train steps 70086
Length 3662
Episode 799 total samples 2256607 train steps 70206
Length 3040
Episode 803 total samples 2259760 train steps 70305
Length 1597
Episode 805 total samples 2262351 train steps 70386
Length 4047
Episode 809 total samples 2264869 train steps 70465
Length 2606
Episode 812 total samples 2268313 train steps 70572
Length 2455
Episode 817 total samples 2271230 train steps 70663
Length 2116
exploration 0.0999991000096
Episode 823 total samples 2274099 train steps 70753
Length 808
Episode 825 total samples 2276636 train steps 70832
Length 1897
Episode 829 total samples 2279646 train steps 70926
Length 1681
Episode 834 total samples 2282332 train steps 71010
Length 1238
Episode 838 total samples 2285278 train steps 71102
Length 3291
Episode 843 total samples 2288534 train steps 71204
Length 2163
Episode 847 total samples 2292051 train steps 71314
Length 2494
Episode 852 total samples 2294891 train steps 71403
Length 1365
Episode 855 total samples 2298715 train steps 71522
Length 4245
exploration 0.0999991000096
Episode 858 total samples 2301228 train steps 71601
Length 2238
Episode 862 total samples 2304414 train steps 71700
Length 2845
Episode 866 total samples 2308103 train steps 71816
Length 2652
Episode 869 total samples 2311154 train steps 71911
Length 2712
Episode 874 total samples 2314340 train steps 72011
Length 1758
Episode 878 total samples 2318373 train steps 72137
Length 3459
Episode 882 total samples 2321295 train steps 72228
Length 1617
Episode 886 total samples 2324233 train steps 72320
Length 1300
exploration 0.0999991000096
Episode 888 total samples 2327203 train steps 72413
Length 1709
Episode 893 total samples 2330441 train steps 72514
Length 3032
Episode 897 total samples 2333832 train steps 72620
Length 2109
Episode 900 total samples 2336938 train steps 72717
Length 1995
Episode 903 total samples 2339962 train steps 72811
Length 1118
Episode 907 total samples 2342941 train steps 72904
Length 1411
Episode 911 total samples 2345856 train steps 72995
Length 1084
Episode 915 total samples 2348688 train steps 73084
Length 1316
exploration 0.0999991000096
Episode 918 total samples 2352397 train steps 73200
Length 3917
Episode 922 total samples 2356669 train steps 73333
Length 3929
Episode 925 total samples 2359665 train steps 73427
Length 1149
Episode 929 total samples 2362680 train steps 73521
Length 1090
Episode 933 total samples 2365498 train steps 73609
Length 982
Episode 937 total samples 2368535 train steps 73704
Length 1114
Episode 940 total samples 2371698 train steps 73803
Length 2081
Episode 944 total samples 2374723 train steps 73898
Length 2397
exploration 0.0999991000096
Episode 946 total samples 2378107 train steps 74003
Length 2427
Episode 949 total samples 2381039 train steps 74095
Length 1140
Episode 952 total samples 2384929 train steps 74217
Length 2877
Episode 957 total samples 2387724 train steps 74304
Length 1625
Episode 961 total samples 2390541 train steps 74392
Length 1246
Episode 964 total samples 2393962 train steps 74499
Length 2048
Episode 966 total samples 2396553 train steps 74580
Length 3920
Episode 968 total samples 2399872 train steps 74683
Length 5257
exploration 0.0999991000096
Episode 972 total samples 2402713 train steps 74772
Length 1410
Episode 975 total samples 2407573 train steps 74924
Length 4919
Episode 978 total samples 2410517 train steps 75016
Length 4365
Episode 981 total samples 2413639 train steps 75114
Length 1290
Episode 983 total samples 2416492 train steps 75203
Length 1521
Episode 985 total samples 2419201 train steps 75288
Length 2481
Episode 989 total samples 2422929 train steps 75404
Length 2918
Episode 992 total samples 2426669 train steps 75521
Length 4487
exploration 0.0999991000096
Episode 997 total samples 2429503 train steps 75609
Length 1455
Episode 1001 total samples 2433303 train steps 75728
Length 3678
Episode 1004 total samples 2436667 train steps 75833
Length 3877
Episode 1007 total samples 2439332 train steps 75917
Length 3362
Episode 1009 total samples 2443189 train steps 76037
Length 3758
Episode 1013 total samples 2445757 train steps 76117
Length 2191
Episode 1016 total samples 2448584 train steps 76206
Length 1017
Episode 1020 total samples 2451130 train steps 76285
Length 1465
exploration 0.0999991000096
Episode 1024 total samples 2454162 train steps 76380
Length 1346
Episode 1028 total samples 2457115 train steps 76472
Length 1048
Episode 1033 total samples 2460089 train steps 76565
Length 958
Episode 1037 total samples 2464026 train steps 76688
Length 3638
Episode 1042 total samples 2467205 train steps 76788
Length 1529
Episode 1046 total samples 2470168 train steps 76880
Length 2215
Episode 1050 total samples 2473391 train steps 76981
Length 2357
Episode 1055 total samples 2476969 train steps 77093
Length 3240
exploration 0.0999991000096
Episode 1057 total samples 2480121 train steps 77191
Length 2205
Episode 1059 total samples 2482999 train steps 77281
Length 1316
Episode 1064 total samples 2485560 train steps 77361
Length 1255
Episode 1068 total samples 2488750 train steps 77461
Length 1724
Episode 1072 total samples 2491875 train steps 77559
Length 1662
Episode 1075 total samples 2494403 train steps 77638
Length 1200
Episode 1079 total samples 2496993 train steps 77719
Length 2131
Episode 1083 total samples 2500111 train steps 77816
Length 2007
Episode 1088 total samples 2503632 train steps 77926
Length 3492
exploration 0.0999991000096
Episode 1092 total samples 2506407 train steps 78013
Length 1261
Episode 1096 total samples 2509576 train steps 78112
Length 1957
Episode 1099 total samples 2512325 train steps 78198
Length 2007
Episode 1103 total samples 2515937 train steps 78311
Length 2874
Episode 1106 total samples 2518500 train steps 78391
Length 1524
Episode 1110 total samples 2521698 train steps 78491
Length 2036
Episode 1114 total samples 2524353 train steps 78574
Length 1525
Episode 1118 total samples 2527140 train steps 78661
Length 1718
Episode 1122 total samples 2530918 train steps 78779
Length 4838
exploration 0.0999991000096
Episode 1126 total samples 2534183 train steps 78881
Length 2195
Episode 1131 total samples 2538201 train steps 79006
Length 3406
Episode 1136 total samples 2541156 train steps 79099
Length 1859
Episode 1139 total samples 2543683 train steps 79178
Length 2428
Episode 1142 total samples 2546193 train steps 79256
Length 2040
Episode 1145 total samples 2548736 train steps 79335
Length 2567
Episode 1147 total samples 2551306 train steps 79416
Length 1486
Episode 1151 total samples 2554256 train steps 79508
Length 2002
exploration 0.0999991000096
Episode 1157 total samples 2557692 train steps 79615
Length 2500
Episode 1161 total samples 2560547 train steps 79705
Length 1619
Episode 1164 total samples 2563696 train steps 79803
Length 1726
Episode 1168 total samples 2567021 train steps 79907
Length 3038
Episode 1173 total samples 2570100 train steps 80003
Length 2296
Episode 1176 total samples 2572649 train steps 80083
Length 2162
Episode 1182 total samples 2575480 train steps 80171
Length 1427
Episode 1186 total samples 2578418 train steps 80263
Length 1736
Episode 1190 total samples 2582672 train steps 80396
Length 4021
exploration 0.0999991000096
Episode 1195 total samples 2585700 train steps 80491
Length 1680
Episode 1199 total samples 2589122 train steps 80598
Length 2272
Episode 1202 total samples 2591800 train steps 80681
Length 1356
Episode 1204 total samples 2594990 train steps 80781
Length 1591
Episode 1207 total samples 2597678 train steps 80865
Length 1463
Episode 1210 total samples 2600542 train steps 80954
Length 4232
Episode 1216 total samples 2603657 train steps 81052
Length 2083
Episode 1218 total samples 2607090 train steps 81159
Length 4941
exploration 0.0999991000096
Episode 1222 total samples 2610305 train steps 81260
Length 1870
Episode 1225 total samples 2612866 train steps 81340
Length 1651
Episode 1229 total samples 2615971 train steps 81437
Length 1699
Episode 1232 total samples 2619101 train steps 81534
Length 2650
Episode 1236 total samples 2622373 train steps 81637
Length 1810
Episode 1237 total samples 2625246 train steps 81726
Length 5746
Episode 1240 total samples 2628417 train steps 81826
Length 1696
Episode 1244 total samples 2631759 train steps 81930
Length 1762
exploration 0.0999991000096
Episode 1249 total samples 2634385 train steps 82012
Length 1042
Episode 1252 total samples 2637759 train steps 82117
Length 3647
Episode 1256 total samples 2640787 train steps 82212
Length 2035
Episode 1260 total samples 2643305 train steps 82291
Length 1093
Episode 1264 total samples 2646001 train steps 82375
Length 1290
Episode 1267 total samples 2649218 train steps 82476
Length 1550
Episode 1270 total samples 2651975 train steps 82562
Length 1159
Episode 1274 total samples 2654525 train steps 82641
Length 746
Episode 1279 total samples 2657132 train steps 82723
Length 950
exploration 0.0999991000096
Episode 1282 total samples 2659925 train steps 82810
Length 782
Episode 1285 total samples 2662458 train steps 82889
Length 639
Episode 1288 total samples 2665239 train steps 82976
Length 960
Episode 1295 total samples 2668513 train steps 83079
Length 1603
Episode 1300 total samples 2671243 train steps 83164
Length 935
Episode 1304 total samples 2674312 train steps 83260
Length 1549
Episode 1308 total samples 2676978 train steps 83343
Length 1900
Episode 1314 total samples 2681044 train steps 83470
Length 3515
Episode 1318 total samples 2684382 train steps 83574
Length 2823
exploration 0.0999991000096
Episode 1323 total samples 2687774 train steps 83680
Length 2177
Episode 1327 total samples 2690825 train steps 83776
Length 1936
Episode 1330 total samples 2693597 train steps 83862
Length 2404
Episode 1334 total samples 2698174 train steps 84005
Length 5539
Episode 1337 total samples 2701067 train steps 84096
Length 2422
Episode 1340 total samples 2705865 train steps 84246
Length 4731
Episode 1343 total samples 2708463 train steps 84327
Length 2195
exploration 0.0999991000096
Episode 1349 total samples 2711275 train steps 84415
Length 1379
Episode 1353 total samples 2715000 train steps 84531
Length 3581
Episode 1358 total samples 2717693 train steps 84615
Length 627
Episode 1361 total samples 2720441 train steps 84701
Length 1323
Episode 1365 total samples 2724050 train steps 84814
Length 4809
Episode 1368 total samples 2726689 train steps 84897
Length 758
Episode 1373 total samples 2729963 train steps 84999
Length 1553
Episode 1376 total samples 2732492 train steps 85078
Length 1835
Episode 1380 total samples 2735547 train steps 85173
Length 1192
exploration 0.0999991000096
Episode 1382 total samples 2738056 train steps 85252
Length 1556
Episode 1387 total samples 2740843 train steps 85339
Length 1684
Episode 1393 total samples 2743414 train steps 85419
Length 1039
Episode 1395 total samples 2746190 train steps 85506
Length 4214
Episode 1398 total samples 2748701 train steps 85584
Length 2475
Episode 1403 total samples 2751722 train steps 85679
Length 1090
Episode 1406 total samples 2754909 train steps 85778
Length 2631
Episode 1409 total samples 2758371 train steps 85887
Length 4144
exploration 0.0999991000096
Episode 1413 total samples 2761038 train steps 85970
Length 554
Episode 1415 total samples 2764094 train steps 86065
Length 4153
Episode 1419 total samples 2767714 train steps 86179
Length 2736
Episode 1420 total samples 2770350 train steps 86261
Length 5271
Episode 1423 total samples 2773212 train steps 86350
Length 2267
Episode 1427 total samples 2776783 train steps 86462
Length 2330
Episode 1431 total samples 2779288 train steps 86540
Length 949
Episode 1437 total samples 2782429 train steps 86638
Length 1884
Episode 1441 total samples 2785099 train steps 86722
Length 502
exploration 0.0999991000096
Episode 1444 total samples 2787756 train steps 86805
Length 3612
Episode 1449 total samples 2790873 train steps 86902
Length 1285
Episode 1453 total samples 2793383 train steps 86981
Length 2119
Episode 1458 total samples 2796161 train steps 87068
Length 695
Episode 1462 total samples 2798748 train steps 87148
Length 1623
Episode 1465 total samples 2802271 train steps 87258
Length 2981
Episode 1469 total samples 2805278 train steps 87352
Length 2264
Episode 1474 total samples 2808335 train steps 87448
Length 1932
Episode 1478 total samples 2811844 train steps 87558
Length 2728
exploration 0.0999991000096
Episode 1483 total samples 2814688 train steps 87646
Length 1201
Episode 1486 total samples 2817392 train steps 87731
Length 1800
Episode 1491 total samples 2819951 train steps 87811
Length 604
Episode 1497 total samples 2822665 train steps 87896
Length 711
Episode 1502 total samples 2826395 train steps 88012
Length 3546
Episode 1504 total samples 2829349 train steps 88105
Length 5162
Episode 1510 total samples 2832950 train steps 88217
Length 2351
Episode 1513 total samples 2837347 train steps 88355
Length 3951
exploration 0.0999991000096
Episode 1515 total samples 2840018 train steps 88438
Length 1523
Episode 1518 total samples 2843006 train steps 88531
Length 1279
Episode 1525 total samples 2845580 train steps 88612
Length 1081
Episode 1527 total samples 2848285 train steps 88696
Length 3260
Episode 1530 total samples 2850891 train steps 88778
Length 2052
Episode 1534 total samples 2853528 train steps 88860
Length 2316
Episode 1539 total samples 2856662 train steps 88958
Length 2452
Episode 1542 total samples 2859380 train steps 89043
Length 2554
Episode 1546 total samples 2862063 train steps 89127
Length 1239
exploration 0.0999991000096
Episode 1549 total samples 2864989 train steps 89218
Length 1133
Episode 1554 total samples 2867770 train steps 89305
Length 706
Episode 1558 total samples 2870686 train steps 89396
Length 3143
Episode 1561 total samples 2873333 train steps 89479
Length 1358
Episode 1564 total samples 2876267 train steps 89571
Length 2693
Episode 1568 total samples 2878962 train steps 89655
Length 3545
Episode 1574 total samples 2882017 train steps 89751
Length 1444
Episode 1578 total samples 2885818 train steps 89869
Length 3284
Episode 1582 total samples 2888582 train steps 89956
Length 1775
exploration 0.0999991000096
Episode 1585 total samples 2891772 train steps 90055
Length 1485
Episode 1588 total samples 2894420 train steps 90138
Length 1509
Episode 1591 total samples 2897431 train steps 90232
Length 2690
Episode 1594 total samples 2900412 train steps 90325
Length 3820
Episode 1598 total samples 2902924 train steps 90404
Length 693
Episode 1600 total samples 2906389 train steps 90512
Length 4609
Episode 1605 total samples 2909185 train steps 90600
Length 1195
Episode 1608 total samples 2911741 train steps 90679
Length 1364
Episode 1611 total samples 2914266 train steps 90758
Length 1647
exploration 0.0999991000096
Episode 1613 total samples 2917480 train steps 90859
Length 3242
Episode 1618 total samples 2920188 train steps 90943
Length 1039
Episode 1621 total samples 2923033 train steps 91032
Length 2137
Episode 1622 total samples 2925941 train steps 91123
Length 5815
Episode 1626 total samples 2928540 train steps 91204
Length 836
Episode 1629 total samples 2931562 train steps 91299
Length 3859
Episode 1632 total samples 2934706 train steps 91397
Length 1946
Episode 1636 total samples 2938017 train steps 91501
Length 1960
exploration 0.0999991000096
Episode 1639 total samples 2941877 train steps 91621
Length 3209
Episode 1643 total samples 2945240 train steps 91726
Length 1905
Episode 1648 total samples 2949340 train steps 91854
Length 3478
Episode 1651 total samples 2952238 train steps 91945
Length 824
Episode 1655 total samples 2954941 train steps 92029
Length 1236
Episode 1660 total samples 2957924 train steps 92123
Length 1328
Episode 1664 total samples 2960515 train steps 92204
Length 1161
Episode 1666 total samples 2963509 train steps 92297
Length 2485
Episode 1670 total samples 2966243 train steps 92383
Length 2761
exploration 0.0999991000096
Episode 1674 total samples 2969189 train steps 92475
Length 1094
Episode 1678 total samples 2972090 train steps 92565
Length 1930
Episode 1679 total samples 2975202 train steps 92663
Length 6224
Episode 1682 total samples 2978159 train steps 92755
Length 1823
Episode 1683 total samples 2980864 train steps 92839
Length 5409
Episode 1685 total samples 2983511 train steps 92922
Length 2251
Episode 1689 total samples 2986907 train steps 93028
Length 3937
Episode 1692 total samples 2989691 train steps 93115
Length 2589
exploration 0.0999991000096
Episode 1696 total samples 2993061 train steps 93221
Length 2062
Episode 1701 total samples 2995785 train steps 93306
Length 1696
Episode 1704 total samples 2998756 train steps 93399
Length 1959
Episode 1709 total samples 3001731 train steps 93492
Length 1698
Episode 1714 total samples 3006959 train steps 93655
Length 5812
Episode 1717 total samples 3010338 train steps 93761
Length 2693
Episode 1720 total samples 3013012 train steps 93844
Length 2121
Episode 1724 total samples 3016106 train steps 93941
Length 1810
exploration 0.0999991000096
Episode 1727 total samples 3019160 train steps 94036
Length 2964
Episode 1731 total samples 3023254 train steps 94164
Length 3852
Episode 1735 total samples 3026374 train steps 94262
Length 1881
Episode 1739 total samples 3029547 train steps 94361
Length 2383
Episode 1741 total samples 3032062 train steps 94439
Length 3583
Episode 1743 total samples 3034660 train steps 94521
Length 1641
Episode 1747 total samples 3038191 train steps 94631
Length 2224
Episode 1751 total samples 3041906 train steps 94747
Length 2712
exploration 0.0999991000096
Episode 1754 total samples 3044657 train steps 94833
Length 2804
Episode 1757 total samples 3047170 train steps 94912
Length 2107
Episode 1760 total samples 3050447 train steps 95014
Length 2900
Episode 1765 total samples 3053286 train steps 95103
Length 1494
Episode 1768 total samples 3056835 train steps 95214
Length 5345
Episode 1771 total samples 3059476 train steps 95296
Length 724
Episode 1774 total samples 3062775 train steps 95399
Length 2237
Episode 1779 total samples 3067620 train steps 95551
Length 5247
exploration 0.0999991000096
Episode 1783 total samples 3070372 train steps 95637
Length 1315
Episode 1786 total samples 3073408 train steps 95731
Length 1342
Episode 1789 total samples 3076556 train steps 95830
Length 4654
Episode 1793 total samples 3079607 train steps 95925
Length 1778
Episode 1797 total samples 3082198 train steps 96006
Length 553
Episode 1801 total samples 3085243 train steps 96101
Length 1878
Episode 1803 total samples 3088187 train steps 96193
Length 4246
Episode 1808 total samples 3091554 train steps 96299
Length 1771
Episode 1812 total samples 3095244 train steps 96414
Length 4099
exploration 0.0999991000096
Episode 1817 total samples 3098740 train steps 96523
Length 2836
Episode 1821 total samples 3102206 train steps 96631
Length 2445
Episode 1823 total samples 3105029 train steps 96720
Length 4575
Episode 1826 total samples 3107998 train steps 96812
Length 2948
Episode 1829 total samples 3111100 train steps 96909
Length 1228
Episode 1834 total samples 3113699 train steps 96991
Length 825
Episode 1840 total samples 3116979 train steps 97093
Length 1723
Episode 1845 total samples 3120495 train steps 97203
Length 2905
exploration 0.0999991000096
Episode 1848 total samples 3123158 train steps 97286
Length 1190
Episode 1852 total samples 3126220 train steps 97382
Length 2150
Episode 1856 total samples 3129599 train steps 97487
Length 3037
Episode 1860 total samples 3133302 train steps 97603
Length 2945
Episode 1866 total samples 3136142 train steps 97692
Length 1158
Episode 1869 total samples 3139550 train steps 97798
Length 2166
Episode 1873 total samples 3142474 train steps 97890
Length 1370
Episode 1877 total samples 3146083 train steps 98003
Length 2572
exploration 0.0999991000096
Episode 1880 total samples 3149115 train steps 98097
Length 2144
Episode 1883 total samples 3151812 train steps 98182
Length 1186
Episode 1886 total samples 3154381 train steps 98262
Length 1986
Episode 1890 total samples 3157444 train steps 98358
Length 1128
Episode 1895 total samples 3162789 train steps 98525
Length 7002
Episode 1898 total samples 3165704 train steps 98616
Length 2169
Episode 1900 total samples 3168482 train steps 98703
Length 3655
Episode 1903 total samples 3171538 train steps 98798
Length 1577
exploration 0.0999991000096
Episode 1910 total samples 3174729 train steps 98898
Length 1869
Episode 1914 total samples 3177445 train steps 98983
Length 899
Episode 1918 total samples 3180218 train steps 99069
Length 1061
Episode 1922 total samples 3183617 train steps 99176
Length 2767
Episode 1927 total samples 3186162 train steps 99255
Length 449
Episode 1930 total samples 3188936 train steps 99342
Length 2231
Episode 1935 total samples 3192124 train steps 99441
Length 1599
Episode 1938 total samples 3195486 train steps 99546
Length 2347
exploration 0.0999991000096
Episode 1943 total samples 3199000 train steps 99656
Length 2817
Episode 1948 total samples 3202006 train steps 99750
Length 1410
Episode 1952 total samples 3204565 train steps 99830
Length 1850
Episode 1955 total samples 3207211 train steps 99913
Length 1337
Episode 1959 total samples 3211350 train steps 100042
Length 3580
done training, I'm tired. I started by 50001
plot not available. Move on.
In [ ]:
car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
Episode 2 total samples 3214351 train steps 100136
Length 1356
Episode 5 total samples 3217580 train steps 100237
Length 2932
Episode 9 total samples 3222182 train steps 100381
Length 4399
Episode 13 total samples 3225249 train steps 100477
Length 1747
Episode 16 total samples 3228175 train steps 100568
Length 2087
Episode 19 total samples 3232630 train steps 100707
Length 4273
Episode 23 total samples 3235733 train steps 100804
Length 1935
exploration 0.0999991000096
Episode 26 total samples 3240089 train steps 100940
Length 6183
Episode 28 total samples 3243681 train steps 101053
Length 5473
Episode 31 total samples 3246320 train steps 101135
Length 1184
Episode 34 total samples 3249417 train steps 101232
Length 3384
Episode 38 total samples 3252730 train steps 101335
Length 2001
Episode 42 total samples 3255529 train steps 101423
Length 1269
Episode 47 total samples 3260501 train steps 101578
Length 5186
Episode 49 total samples 3263677 train steps 101677
Length 4426
exploration 0.0999991000096
Episode 53 total samples 3266620 train steps 101769
Length 885
Episode 55 total samples 3269370 train steps 101855
Length 2026
Episode 59 total samples 3271902 train steps 101934
Length 1823
Episode 63 total samples 3274775 train steps 102024
Length 831
Episode 66 total samples 3277366 train steps 102105
Length 3491
Episode 71 total samples 3280392 train steps 102200
Length 1289
Episode 74 total samples 3283346 train steps 102292
Length 1806
Episode 77 total samples 3286784 train steps 102399
Length 2737
Episode 80 total samples 3289878 train steps 102496
Length 2656
exploration 0.0999991000096
Episode 84 total samples 3292731 train steps 102585
Length 2865
Episode 88 total samples 3295537 train steps 102673
Length 1349
Episode 91 total samples 3299934 train steps 102810
Length 5991
Episode 94 total samples 3302627 train steps 102895
Length 1629
Episode 98 total samples 3305734 train steps 102992
Length 2582
Episode 102 total samples 3309338 train steps 103104
Length 4243
Episode 108 total samples 3313445 train steps 103233
Length 3442
exploration 0.0999991000096
Episode 111 total samples 3316461 train steps 103327
Length 2198
Episode 113 total samples 3319750 train steps 103430
Length 3503
Episode 117 total samples 3322396 train steps 103512
Length 1402
Episode 121 total samples 3325420 train steps 103607
Length 1711
Episode 123 total samples 3328448 train steps 103701
Length 2551
Episode 127 total samples 3331594 train steps 103800
Length 2268
Episode 130 total samples 3335127 train steps 103910
Length 2089
Episode 133 total samples 3338657 train steps 104021
Length 3253
exploration 0.0999991000096
Episode 136 total samples 3341736 train steps 104117
Length 2142
Episode 139 total samples 3345722 train steps 104241
Length 3276
Episode 142 total samples 3348541 train steps 104329
Length 986
Episode 146 total samples 3352173 train steps 104443
Length 2641
Episode 149 total samples 3355410 train steps 104544
Length 2410
Episode 155 total samples 3358432 train steps 104638
Length 1199
Episode 158 total samples 3361682 train steps 104740
Length 1837
Episode 161 total samples 3364709 train steps 104835
Length 1897
exploration 0.0999991000096
Episode 165 total samples 3367433 train steps 104920
Length 972
Episode 169 total samples 3370912 train steps 105028
Length 2263
Episode 172 total samples 3373741 train steps 105117
Length 1705
Episode 176 total samples 3376724 train steps 105210
Length 2620
Episode 180 total samples 3379570 train steps 105299
Length 918
Episode 182 total samples 3382182 train steps 105381
Length 3739
Episode 185 total samples 3385031 train steps 105470
Length 4039
Episode 190 total samples 3388489 train steps 105578
Length 2124
Episode 194 total samples 3391141 train steps 105661
Length 1606
exploration 0.0999991000096
Episode 197 total samples 3393755 train steps 105742
Length 886
Episode 199 total samples 3396343 train steps 105823
Length 3823
Episode 204 total samples 3399743 train steps 105929
Length 1799
Episode 207 total samples 3402254 train steps 106008
Length 1521
Episode 210 total samples 3405070 train steps 106096
Length 1487
Episode 214 total samples 3407692 train steps 106178
Length 543
Episode 217 total samples 3410809 train steps 106275
Length 3672
Episode 219 total samples 3414031 train steps 106376
Length 5104
Episode 222 total samples 3418222 train steps 106507
Length 4602
exploration 0.0999991000096
Episode 227 total samples 3421215 train steps 106600
Length 1797
Episode 231 total samples 3423885 train steps 106684
Length 830
Episode 235 total samples 3427006 train steps 106781
Length 2470
Episode 238 total samples 3429535 train steps 106860
Length 3546
Episode 241 total samples 3432368 train steps 106949
Length 2544
Episode 243 total samples 3436217 train steps 107069
Length 3760
Episode 246 total samples 3438889 train steps 107153
Length 583
Episode 250 total samples 3442322 train steps 107260
Length 2262
exploration 0.0999991000096
Episode 252 total samples 3445038 train steps 107345
Length 2002
Episode 256 total samples 3448105 train steps 107441
Length 1632
Episode 258 total samples 3450812 train steps 107525
Length 2533
Episode 262 total samples 3453485 train steps 107609
Length 530
Episode 265 total samples 3456850 train steps 107714
Length 1975
Episode 268 total samples 3459370 train steps 107793
Length 1706
Episode 271 total samples 3462103 train steps 107878
Length 1905
Episode 275 total samples 3465393 train steps 107981
Length 2009
Episode 278 total samples 3468262 train steps 108071
Length 1211
exploration 0.0999991000096
Episode 282 total samples 3471232 train steps 108163
Length 2580
Episode 285 total samples 3474084 train steps 108253
Length 3364
Episode 288 total samples 3477016 train steps 108344
Length 3176
Episode 292 total samples 3480240 train steps 108445
Length 2465
Episode 295 total samples 3483291 train steps 108540
Length 3240
Episode 298 total samples 3485991 train steps 108625
Length 1549
Episode 301 total samples 3489114 train steps 108722
Length 1894
Episode 305 total samples 3492958 train steps 108842
Length 2920
exploration 0.0999991000096
Episode 307 total samples 3496441 train steps 108951
Length 2674
Episode 311 total samples 3499124 train steps 109035
Length 2020
Episode 314 total samples 3503708 train steps 109178
Length 5306
Episode 319 total samples 3507394 train steps 109294
Length 2791
Episode 323 total samples 3510978 train steps 109406
Length 2960
Episode 327 total samples 3514640 train steps 109520
Length 2360
Episode 329 total samples 3517743 train steps 109617
Length 2645
exploration 0.0999991000096
Episode 333 total samples 3521095 train steps 109722
Length 1747
Episode 337 total samples 3523835 train steps 109807
Length 765
Episode 341 total samples 3526466 train steps 109890
Length 2029
Episode 346 total samples 3529027 train steps 109970
Length 601
Episode 348 total samples 3531726 train steps 110054
Length 1215
Episode 353 total samples 3534287 train steps 110134
Length 753
Episode 357 total samples 3537164 train steps 110224
Length 2373
Episode 360 total samples 3539981 train steps 110312
Length 1692
Episode 363 total samples 3543270 train steps 110415
Length 4768
Episode 365 total samples 3545832 train steps 110495
Length 1585
exploration 0.0999991000096
Episode 368 total samples 3548809 train steps 110588
Length 1648
Episode 371 total samples 3551982 train steps 110687
Length 2042
Episode 374 total samples 3554657 train steps 110771
Length 2261
Episode 377 total samples 3557882 train steps 110871
Length 2075
Episode 381 total samples 3560972 train steps 110968
Length 1207
Episode 383 total samples 3563504 train steps 111047
Length 1252
Episode 387 total samples 3566390 train steps 111137
Length 2210
Episode 392 total samples 3569661 train steps 111239
Length 1859
exploration 0.0999991000096
Episode 395 total samples 3573399 train steps 111356
Length 3421
Episode 399 total samples 3576192 train steps 111443
Length 2109
Episode 402 total samples 3578950 train steps 111530
Length 996
Episode 405 total samples 3582018 train steps 111626
Length 1891
Episode 409 total samples 3584934 train steps 111717
Length 1263
Episode 414 total samples 3587848 train steps 111808
Length 1024
Episode 419 total samples 3591631 train steps 111926
Length 3372
Episode 424 total samples 3594431 train steps 112013
Length 1853
exploration 0.0999991000096
Episode 426 total samples 3597143 train steps 112098
Length 2078
Episode 430 total samples 3600051 train steps 112189
Length 1032
Episode 435 total samples 3602709 train steps 112272
Length 802
Episode 439 total samples 3607243 train steps 112414
Length 4946
Episode 442 total samples 3610410 train steps 112513
Length 3334
Episode 444 total samples 3612962 train steps 112593
Length 3793
Episode 449 total samples 3615953 train steps 112686
Length 1012
Episode 455 total samples 3619321 train steps 112791
Length 1780
exploration 0.0999991000096
Episode 459 total samples 3621923 train steps 112873
Length 603
Episode 462 total samples 3624615 train steps 112957
Length 1902
Episode 465 total samples 3627186 train steps 113037
Length 1191
Episode 470 total samples 3630344 train steps 113136
Length 1916
Episode 475 total samples 3633387 train steps 113231
Length 1566
Episode 479 total samples 3635949 train steps 113311
Length 550
Episode 484 total samples 3638665 train steps 113396
Length 1115
Episode 488 total samples 3641368 train steps 113480
Length 2055
Episode 493 total samples 3644318 train steps 113572
Length 2012
Episode 498 total samples 3646827 train steps 113651
Length 1105
exploration 0.0999991000096
Episode 502 total samples 3649683 train steps 113740
Length 1719
Episode 505 total samples 3652215 train steps 113819
Length 675
Episode 508 total samples 3655037 train steps 113907
Length 2568
Episode 512 total samples 3659222 train steps 114038
Length 3491
Episode 514 total samples 3661862 train steps 114121
Length 1655
Episode 516 total samples 3664439 train steps 114201
Length 3134
Episode 518 total samples 3667466 train steps 114296
Length 3673
Episode 523 total samples 3670195 train steps 114381
Length 1475
Episode 526 total samples 3673828 train steps 114495
Length 4139
exploration 0.0999991000096
Episode 529 total samples 3677309 train steps 114603
Length 3500
Episode 532 total samples 3679977 train steps 114687
Length 954
Episode 537 total samples 3682481 train steps 114765
Length 703
Episode 540 total samples 3685333 train steps 114854
Length 3123
Episode 544 total samples 3688832 train steps 114963
Length 2347
Episode 547 total samples 3691755 train steps 115055
Length 3093
Episode 550 total samples 3695292 train steps 115165
Length 3858
Episode 553 total samples 3698107 train steps 115253
Length 2375
exploration 0.0999991000096
Episode 556 total samples 3700672 train steps 115333
Length 1792
Episode 559 total samples 3703376 train steps 115418
Length 1593
Episode 564 total samples 3705921 train steps 115498
Length 1097
Episode 567 total samples 3708673 train steps 115584
Length 1477
Episode 571 total samples 3711561 train steps 115674
Length 787
Episode 575 total samples 3714718 train steps 115772
Length 2796
Episode 579 total samples 3717879 train steps 115871
Length 2344
Episode 582 total samples 3720767 train steps 115961
Length 3012
Episode 584 total samples 3723351 train steps 116042
Length 2602
exploration 0.0999991000096
Episode 588 total samples 3729393 train steps 116231
Length 7796
Episode 591 total samples 3731927 train steps 116310
Length 2229
Episode 595 total samples 3734474 train steps 116390
Length 614
Episode 598 total samples 3737325 train steps 116479
Length 4267
Episode 602 total samples 3740475 train steps 116577
Length 1771
Episode 606 total samples 3743800 train steps 116681
Length 2014
Episode 609 total samples 3746543 train steps 116767
Length 1698
Episode 612 total samples 3749938 train steps 116873
Length 3307
exploration 0.0999991000096
Episode 614 total samples 3752444 train steps 116951
Length 2211
Episode 618 total samples 3756046 train steps 117064
Length 2380
Episode 623 total samples 3759005 train steps 117156
Length 1083
Episode 627 total samples 3762102 train steps 117253
Length 1552
Episode 631 total samples 3765078 train steps 117346
Length 965
Episode 634 total samples 3768337 train steps 117448
Length 2451
Episode 639 total samples 3771582 train steps 117549
Length 1813
Episode 642 total samples 3775083 train steps 117659
Length 4310
exploration 0.0999991000096
Episode 645 total samples 3778301 train steps 117759
Length 2823
Episode 646 total samples 3780994 train steps 117844
Length 5386
Episode 648 total samples 3783886 train steps 117934
Length 3069
Episode 651 total samples 3786938 train steps 118029
Length 2781
Episode 654 total samples 3790154 train steps 118130
Length 2903
Episode 658 total samples 3792699 train steps 118209
Length 1238
Episode 660 total samples 3795379 train steps 118293
Length 2559
Episode 663 total samples 3799842 train steps 118433
Length 4493
exploration 0.0999991000096
Episode 666 total samples 3802645 train steps 118520
Length 2298
Episode 670 total samples 3806224 train steps 118632
Length 3108
Episode 674 total samples 3809365 train steps 118730
Length 2064
Episode 677 total samples 3812678 train steps 118834
Length 3371
Episode 681 total samples 3815732 train steps 118929
Length 2341
Episode 684 total samples 3818241 train steps 119008
Length 1369
Episode 688 total samples 3821505 train steps 119110
Length 2392
Episode 693 total samples 3825910 train steps 119247
Length 3843
exploration 0.0999991000096
Episode 697 total samples 3828700 train steps 119334
Length 1152
Episode 702 total samples 3832522 train steps 119454
Length 2675
Episode 705 total samples 3835333 train steps 119542
Length 4326
Episode 710 total samples 3838542 train steps 119642
Length 1681
Episode 714 total samples 3842516 train steps 119766
Length 3193
Episode 719 total samples 3845145 train steps 119848
Length 531
Episode 723 total samples 3847805 train steps 119931
Length 1328
Episode 729 total samples 3850854 train steps 120027
Length 1325
exploration 0.0999991000096
Episode 734 total samples 3853484 train steps 120109
Length 721
Episode 738 total samples 3857051 train steps 120220
Length 2523
Episode 740 total samples 3859654 train steps 120302
Length 3261
Episode 743 total samples 3862714 train steps 120397
Length 2843
Episode 747 total samples 3865822 train steps 120494
Length 2801
Episode 749 total samples 3868863 train steps 120589
Length 2551
Episode 752 total samples 3871451 train steps 120670
Length 1577
Episode 757 total samples 3874309 train steps 120760
Length 1027
exploration 0.0999991000096
Episode 761 total samples 3877616 train steps 120863
Length 2072
Episode 766 total samples 3880701 train steps 120959
Length 1437
Episode 769 total samples 3883614 train steps 121050
Length 3320
Episode 771 total samples 3886132 train steps 121129
Length 2639
Episode 774 total samples 3888733 train steps 121210
Length 1018
Episode 777 total samples 3891459 train steps 121296
Length 2546
Episode 780 total samples 3894026 train steps 121376
Length 1092
Episode 782 total samples 3896703 train steps 121459
Length 1850
Episode 786 total samples 3899226 train steps 121538
Length 606
Episode 789 total samples 3902145 train steps 121630
Length 2744
exploration 0.0999991000096
Episode 793 total samples 3905663 train steps 121739
Length 2048
Episode 796 total samples 3908727 train steps 121835
Length 1725
Episode 801 total samples 3912445 train steps 121951
Length 4274
Episode 805 total samples 3916241 train steps 122070
Length 2898
Episode 808 total samples 3919970 train steps 122187
Length 2591
Episode 811 total samples 3922816 train steps 122275
Length 2075
Episode 814 total samples 3925456 train steps 122358
Length 433
exploration 0.0999991000096
Episode 819 total samples 3927960 train steps 122436
Length 619
Episode 823 total samples 3931071 train steps 122533
Length 1346
Episode 829 total samples 3933682 train steps 122615
Length 1609
Episode 831 total samples 3936547 train steps 122705
Length 2934
Episode 835 total samples 3939152 train steps 122786
Length 1808
Episode 837 total samples 3941970 train steps 122874
Length 1657
Episode 840 total samples 3944979 train steps 122968
Length 2387
Episode 845 total samples 3947860 train steps 123058
Length 1846
Episode 850 total samples 3950557 train steps 123142
Length 821
exploration 0.0999991000096
Episode 854 total samples 3953324 train steps 123229
Length 938
Episode 859 total samples 3955825 train steps 123307
Length 866
Episode 863 total samples 3958330 train steps 123385
Length 846
Episode 867 total samples 3962274 train steps 123509
Length 3910
Episode 871 total samples 3964793 train steps 123587
Length 1923
Episode 876 total samples 3968121 train steps 123691
Length 2679
Episode 881 total samples 3971939 train steps 123811
Length 3026
Episode 884 total samples 3974647 train steps 123895
Length 3023
Episode 890 total samples 3977503 train steps 123984
Length 999
exploration 0.0999991000096
Episode 892 total samples 3980567 train steps 124080
Length 3945
Episode 895 total samples 3983332 train steps 124167
Length 1336
Episode 900 total samples 3987514 train steps 124297
Length 4059
Episode 905 total samples 3990916 train steps 124404
Length 2532
Episode 910 total samples 3994757 train steps 124524
Length 3171
Episode 912 total samples 3997502 train steps 124609
Length 1801
Episode 914 total samples 4001762 train steps 124743
Length 3967
exploration 0.0999991000096
Episode 918 total samples 4004279 train steps 124821
Length 741
Episode 922 total samples 4007104 train steps 124909
Length 1482
Episode 925 total samples 4009787 train steps 124993
Length 2431
Episode 928 total samples 4012649 train steps 125083
Length 1296
Episode 933 total samples 4015540 train steps 125173
Length 966
Episode 936 total samples 4019771 train steps 125305
Length 3836
Episode 940 total samples 4022731 train steps 125398
Length 1179
Episode 944 total samples 4026084 train steps 125503
Length 3369
Episode 947 total samples 4028834 train steps 125589
Length 2301
exploration 0.0999991000096
Episode 951 total samples 4032270 train steps 125696
Length 1913
Episode 954 total samples 4035200 train steps 125787
Length 2894
Episode 957 total samples 4038124 train steps 125879
Length 1165
Episode 960 total samples 4040740 train steps 125961
Length 813
Episode 964 total samples 4043613 train steps 126050
Length 1200
Episode 966 total samples 4046269 train steps 126133
Length 3552
Episode 970 total samples 4049479 train steps 126234
Length 3122
Episode 973 total samples 4052080 train steps 126315
Length 2704
Episode 975 total samples 4055365 train steps 126418
Length 2724
exploration 0.0999991000096
Episode 978 total samples 4058008 train steps 126500
Length 2529
Episode 980 total samples 4060577 train steps 126581
Length 675
Episode 983 total samples 4063179 train steps 126662
Length 2195
Episode 986 total samples 4066448 train steps 126764
Length 4309
Episode 990 total samples 4069358 train steps 126855
Length 1622
Episode 996 total samples 4072636 train steps 126957
Length 1606
Episode 999 total samples 4075834 train steps 127057
Length 4203
Episode 1002 total samples 4078883 train steps 127153
Length 1966
exploration 0.0999991000096
Episode 1006 total samples 4081501 train steps 127234
Length 1058
Episode 1010 total samples 4084579 train steps 127331
Length 2339
Episode 1014 total samples 4087922 train steps 127435
Length 2531
Episode 1018 total samples 4090606 train steps 127519
Length 1722
Episode 1022 total samples 4093390 train steps 127606
Length 1899
Episode 1024 total samples 4096139 train steps 127692
Length 672
Episode 1028 total samples 4099798 train steps 127806
Length 2788
Episode 1033 total samples 4103236 train steps 127914
Length 1883
Episode 1036 total samples 4105910 train steps 127997
Length 1416
exploration 0.0999991000096
Episode 1041 total samples 4108908 train steps 128091
Length 1673
Episode 1043 total samples 4112066 train steps 128190
Length 2664
Episode 1047 total samples 4116354 train steps 128324
Length 4308
Episode 1051 total samples 4119341 train steps 128417
Length 1255
Episode 1056 total samples 4122430 train steps 128513
Length 1476
Episode 1059 total samples 4125399 train steps 128606
Length 2002
Episode 1062 total samples 4128736 train steps 128710
Length 1903
Episode 1066 total samples 4131469 train steps 128796
Length 842
exploration 0.0999991000096
Episode 1071 total samples 4134223 train steps 128882
Length 897
Episode 1075 total samples 4136954 train steps 128967
Length 1019
Episode 1079 total samples 4140836 train steps 129089
Length 3057
Episode 1082 total samples 4144143 train steps 129192
Length 1909
Episode 1085 total samples 4146650 train steps 129270
Length 998
Episode 1089 total samples 4149629 train steps 129363
Length 985
Episode 1092 total samples 4154242 train steps 129508
Length 4605
Episode 1097 total samples 4157053 train steps 129595
Length 1507
exploration 0.0999991000096
Episode 1101 total samples 4160299 train steps 129697
Length 1626
Episode 1106 total samples 4163055 train steps 129783
Length 624
Episode 1108 total samples 4165617 train steps 129863
Length 2488
Episode 1111 total samples 4168240 train steps 129945
Length 3151
Episode 1114 total samples 4171076 train steps 130034
Length 956
Episode 1118 total samples 4173991 train steps 130125
Length 1206
Episode 1122 total samples 4176777 train steps 130212
Length 818
Episode 1124 total samples 4180518 train steps 130329
Length 5726
exploration 0.0999991000096
Episode 1127 total samples 4183070 train steps 130408
Length 1847
Episode 1131 total samples 4186734 train steps 130523
Length 3172
Episode 1135 total samples 4189496 train steps 130609
Length 1866
Episode 1139 total samples 4192620 train steps 130707
Length 1370
Episode 1144 total samples 4196214 train steps 130819
Length 2249
Episode 1148 total samples 4198747 train steps 130898
Length 433
Episode 1151 total samples 4201974 train steps 130999
Length 2180
Episode 1155 total samples 4205022 train steps 131094
Length 1981
Episode 1158 total samples 4207810 train steps 131182
Length 2276
exploration 0.0999991000096
Episode 1163 total samples 4211001 train steps 131281
Length 1767
Episode 1168 total samples 4214371 train steps 131387
Length 1762
Episode 1171 total samples 4217657 train steps 131489
Length 2405
Episode 1174 total samples 4220618 train steps 131582
Length 3566
Episode 1177 total samples 4223401 train steps 131669
Length 1852
Episode 1180 total samples 4226014 train steps 131750
Length 2690
Episode 1184 total samples 4229010 train steps 131844
Length 1921
Episode 1187 total samples 4231836 train steps 131932
Length 3089
exploration 0.0999991000096
Episode 1193 total samples 4235135 train steps 132035
Length 1608
Episode 1197 total samples 4238183 train steps 132131
Length 2375
Episode 1200 total samples 4240835 train steps 132214
Length 1929
Episode 1204 total samples 4244739 train steps 132336
Length 3156
Episode 1209 total samples 4248007 train steps 132438
Length 2028
Episode 1213 total samples 4251929 train steps 132560
Length 3055
Episode 1219 total samples 4254614 train steps 132644
Length 779
Episode 1222 total samples 4257469 train steps 132733
Length 2898
Episode 1223 total samples 4260701 train steps 132834
Length 6464
exploration 0.0999991000096
Episode 1227 total samples 4264031 train steps 132938
Length 2181
Episode 1233 total samples 4266921 train steps 133029
Length 1300
Episode 1237 total samples 4269578 train steps 133112
Length 1789
Episode 1239 total samples 4273243 train steps 133226
Length 5413
Episode 1242 total samples 4276431 train steps 133326
Length 1532
Episode 1246 total samples 4280992 train steps 133468
Length 5236
Episode 1250 total samples 4283590 train steps 133550
Length 1090
exploration 0.0999991000096
Episode 1254 total samples 4286456 train steps 133639
Length 1181
Episode 1257 total samples 4289338 train steps 133729
Length 1282
Episode 1260 total samples 4292129 train steps 133817
Length 2892
Episode 1264 total samples 4295236 train steps 133914
Length 3573
Episode 1269 total samples 4298993 train steps 134031
Length 2673
Episode 1272 total samples 4301584 train steps 134112
Length 2898
Episode 1274 total samples 4304146 train steps 134192
Length 2788
Episode 1279 total samples 4307077 train steps 134284
Length 1149
Episode 1284 total samples 4310308 train steps 134385
Length 2001
exploration 0.0999991000096
Episode 1290 total samples 4313031 train steps 134470
Length 636
Episode 1292 total samples 4316010 train steps 134563
Length 3914
Episode 1295 total samples 4319273 train steps 134665
Length 3260
Episode 1299 total samples 4322244 train steps 134758
Length 1787
Episode 1302 total samples 4325074 train steps 134846
Length 2534
Episode 1305 total samples 4327608 train steps 134925
Length 452
Episode 1308 total samples 4330133 train steps 135004
Length 2140
Episode 1311 total samples 4333295 train steps 135103
Length 2242
Episode 1314 total samples 4336088 train steps 135190
Length 1441
exploration 0.0999991000096
Episode 1318 total samples 4339187 train steps 135287
Length 1788
Episode 1321 total samples 4342324 train steps 135385
Length 1421
Episode 1326 total samples 4345473 train steps 135484
Length 2758
Episode 1329 total samples 4349658 train steps 135614
Length 5692
Episode 1330 total samples 4352278 train steps 135696
Length 5239
Episode 1335 total samples 4355057 train steps 135783
Length 1365
Episode 1339 total samples 4357625 train steps 135863
Length 1329
Episode 1341 total samples 4360450 train steps 135952
Length 4802
exploration 0.0999991000096
Episode 1344 total samples 4362984 train steps 136031
Length 598
Episode 1346 total samples 4367263 train steps 136164
Length 3711
Episode 1351 total samples 4371112 train steps 136285
Length 3336
Episode 1355 total samples 4373756 train steps 136367
Length 564
Episode 1358 total samples 4376516 train steps 136454
Length 3009
Episode 1361 total samples 4379268 train steps 136540
Length 1357
Episode 1364 total samples 4382052 train steps 136627
Length 1320
Episode 1367 total samples 4384614 train steps 136707
Length 2629
Episode 1371 total samples 4387920 train steps 136810
Length 2073
exploration 0.0999991000096
Episode 1374 total samples 4391078 train steps 136909
Length 1896
Episode 1378 total samples 4394062 train steps 137002
Length 2560
Episode 1380 total samples 4396719 train steps 137085
Length 2301
Episode 1382 total samples 4399253 train steps 137164
Length 918
Episode 1386 total samples 4403022 train steps 137282
Length 2539
Episode 1389 total samples 4406098 train steps 137378
Length 2284
Episode 1392 total samples 4408651 train steps 137458
Length 1912
Episode 1395 total samples 4411828 train steps 137557
Length 2016
exploration 0.0999991000096
Episode 1399 total samples 4414919 train steps 137654
Length 1833
Episode 1404 total samples 4418043 train steps 137751
Length 1744
Episode 1407 total samples 4420908 train steps 137841
Length 1548
Episode 1410 total samples 4423881 train steps 137934
Length 1697
Episode 1413 total samples 4426622 train steps 138019
Length 2006
Episode 1417 total samples 4429949 train steps 138123
Length 2295
Episode 1420 total samples 4432927 train steps 138216
Length 2804
Episode 1425 total samples 4435479 train steps 138296
Length 800
Episode 1430 total samples 4438091 train steps 138378
Length 400
exploration 0.0999991000096
Episode 1434 total samples 4440595 train steps 138456
Length 444
Episode 1439 total samples 4443651 train steps 138552
Length 1287
Episode 1443 total samples 4447289 train steps 138665
Length 2476
Episode 1446 total samples 4449975 train steps 138749
Length 2182
Episode 1448 total samples 4452658 train steps 138833
Length 597
Episode 1451 total samples 4455381 train steps 138918
Length 3250
Episode 1453 total samples 4458143 train steps 139004
Length 2553
Episode 1457 total samples 4461091 train steps 139097
Length 897
exploration 0.0999991000096
Episode 1460 total samples 4463786 train steps 139181
Length 1148
Episode 1464 total samples 4466832 train steps 139276
Length 1735
Episode 1467 total samples 4469878 train steps 139371
Length 3138
Episode 1469 total samples 4473779 train steps 139493
Length 5113
Episode 1473 total samples 4476944 train steps 139592
Length 1741
Episode 1476 total samples 4479648 train steps 139676
Length 741
Episode 1480 total samples 4482587 train steps 139768
Length 975
Episode 1484 total samples 4485115 train steps 139847
Length 1011
Episode 1487 total samples 4488139 train steps 139942
Length 1544
exploration 0.0999991000096
Episode 1491 total samples 4490741 train steps 140023
Length 2028
Episode 1494 total samples 4493396 train steps 140106
Length 918
Episode 1499 total samples 4496828 train steps 140213
Length 2151
Episode 1503 total samples 4500052 train steps 140314
Length 1539
Episode 1507 total samples 4503431 train steps 140420
Length 2488
Episode 1511 total samples 4506401 train steps 140513
Length 1760
Episode 1517 total samples 4509166 train steps 140599
Length 977
Episode 1520 total samples 4511746 train steps 140680
Length 1214
Episode 1524 total samples 4514787 train steps 140775
Length 1681
exploration 0.0999991000096
Episode 1527 total samples 4518430 train steps 140888
Length 4092
Episode 1529 total samples 4521023 train steps 140969
Length 3924
Episode 1533 total samples 4524584 train steps 141081
Length 2776
In [ ]:
car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
In [1]:
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
In [2]:
from dqn_learning import *
# import cPickle
# %load_ext autoreload
# %autoreload 2
%matplotlib inline
# import numpy as np
[2016-08-25 14:19:03,547] Site environment registry incorrect: Scoreboard did not register all envs: set(['AcrobotContinuous-v0'])
In [3]:
car = q_learning(gamma=1.,
lambda_=0.9,
# init_epsilon = 0.0,
# update_epsilon = False,
init_epsilon = 1.0,
end_epsilon = 0.1,
update_epsilon = True,
exploration_decrease_length = 1e6,
# N_0= 50,
environment = 'Acrobot-v0',
plot_resolution = 30,
nn_size_hidden = [400,300], # [100,100] --> 0.01, [100,100,100] --> similar [400,300]
nn_batch_size = 128,
nn_learning_rate = 2.5e-4,
replay_memory_size = 3e5, # larger than nn_batch_size*1e3 or doesn't train
descent_method = 'adam',
# dropout_keep_prob = 0.5,
ema_decay_rate = 0.999,
train_frequency = 4.0,
from_pixels = True,
reg_weight = 0.0,
do_pretrain = False,
pretrain_steps = 0
)
[2016-08-25 14:19:06,741] Making new env: Acrobot-v0
conv 128 8 8 2
weights std = 0.125
[8, 8, 2, 32]
bias std = 0.0
conv 512 4 4 32
weights std = 0.0625
[4, 4, 32, 64]
bias std = 0.0
conv 576 3 3 64
weights std = 0.0589255650989
[3, 3, 64, 64]
bias std = 0.0
fc 7744
weights std = 0.0160706086633
bias std = 0.0
fc 512
weights std = 0.0441941738242
bias std = 0.0
Q-learning configured in nn graph
descending with adam
using environment Acrobot-v0
qnn target q-learning False False
In [4]:
car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
Episode 25 total samples 10459 train steps 14
Length 901
Episode 31 total samples 13133 train steps 98
Length 1597
Episode 34 total samples 15889 train steps 184
Length 1707
Episode 39 total samples 19115 train steps 285
Length 2470
Episode 47 total samples 22100 train steps 378
Length 1241
Episode 51 total samples 24831 train steps 463
Length 827
exploration 0.977322700001
/usr/lib/pymodules/python2.7/matplotlib/collections.py:548: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
if self._edgecolors == 'face':
Episode 56 total samples 27741 train steps 554
Length 1492
Episode 61 total samples 30327 train steps 635
Length 576
Episode 67 total samples 33453 train steps 733
Length 1255
Episode 72 total samples 36508 train steps 828
Length 2049
Episode 76 total samples 39123 train steps 910
Length 358
Episode 82 total samples 41834 train steps 995
Length 561
Episode 88 total samples 44951 train steps 1092
Length 1543
Episode 94 total samples 47478 train steps 1171
Length 858
Episode 100 total samples 50085 train steps 1253
Length 872
exploration 0.954159400002
Episode 103 total samples 52981 train steps 1343
Length 3038
Episode 109 total samples 55731 train steps 1429
Length 1098
Episode 113 total samples 58570 train steps 1518
Length 1003
Episode 118 total samples 61245 train steps 1601
Length 1196
Episode 125 total samples 67348 train steps 1792
Length 7936
Episode 129 total samples 69889 train steps 1872
Length 1505
Episode 138 total samples 72860 train steps 1964
Length 1191
Episode 142 total samples 76360 train steps 2074
Length 3674
exploration 0.931276000002
Episode 147 total samples 79298 train steps 2166
Length 2203
Episode 153 total samples 82837 train steps 2276
Length 2212
Episode 157 total samples 85362 train steps 2355
Length 2477
Episode 163 total samples 88124 train steps 2441
Length 719
Episode 169 total samples 91355 train steps 2542
Length 1727
Episode 174 total samples 94136 train steps 2629
Length 1271
Episode 180 total samples 96684 train steps 2709
Length 1807
Episode 186 total samples 99465 train steps 2796
Length 1131
Episode 191 total samples 102551 train steps 2892
Length 2879
exploration 0.907704100003
Episode 197 total samples 105062 train steps 2971
Length 1610
Episode 204 total samples 107992 train steps 3062
Length 1809
Episode 208 total samples 110560 train steps 3142
Length 1194
Episode 213 total samples 113137 train steps 3223
Length 650
Episode 219 total samples 115759 train steps 3305
Length 869
Episode 225 total samples 118295 train steps 3384
Length 1186
Episode 230 total samples 121107 train steps 3472
Length 836
Episode 237 total samples 123810 train steps 3557
Length 730
Episode 242 total samples 126597 train steps 3644
Length 631
exploration 0.884903500004
Episode 247 total samples 130445 train steps 3764
Length 3284
Episode 254 total samples 133332 train steps 3854
Length 1263
Episode 260 total samples 136038 train steps 3939
Length 436
Episode 265 total samples 138840 train steps 4026
Length 1393
Episode 272 total samples 141653 train steps 4114
Length 875
Episode 276 total samples 144168 train steps 4193
Length 1004
Episode 280 total samples 146731 train steps 4273
Length 950
Episode 287 total samples 149541 train steps 4361
Length 1169
Episode 295 total samples 152245 train steps 4445
Length 661
exploration 0.861331600005
Episode 299 total samples 155118 train steps 4535
Length 1141
Episode 304 total samples 157765 train steps 4618
Length 1002
Episode 307 total samples 160731 train steps 4710
Length 1380
Episode 313 total samples 163339 train steps 4792
Length 1105
Episode 321 total samples 166158 train steps 4880
Length 802
Episode 328 total samples 168659 train steps 4958
Length 880
Episode 333 total samples 171174 train steps 5037
Length 1056
Episode 340 total samples 174169 train steps 5130
Length 1297
Episode 347 total samples 176688 train steps 5209
Length 659
Episode 352 total samples 179208 train steps 5288
Length 1418
exploration 0.838712800005
Episode 357 total samples 182181 train steps 5381
Length 1265
Episode 363 total samples 184701 train steps 5459
Length 281
Episode 369 total samples 187410 train steps 5544
Length 458
Episode 376 total samples 189922 train steps 5623
Length 523
Episode 382 total samples 193718 train steps 5741
Length 2639
Episode 389 total samples 196492 train steps 5828
Length 762
Episode 394 total samples 199333 train steps 5917
Length 1395
Episode 399 total samples 202547 train steps 6017
Length 2090
exploration 0.815932000006
Episode 406 total samples 205596 train steps 6112
Length 1226
Episode 412 total samples 208533 train steps 6204
Length 1890
Episode 417 total samples 211091 train steps 6284
Length 1075
Episode 421 total samples 213691 train steps 6365
Length 1907
Episode 427 total samples 216352 train steps 6448
Length 595
Episode 434 total samples 219104 train steps 6534
Length 655
Episode 442 total samples 222068 train steps 6627
Length 964
Episode 449 total samples 224750 train steps 6711
Length 1257
Episode 454 total samples 227417 train steps 6794
Length 1306
exploration 0.793252900007
Episode 459 total samples 230137 train steps 6879
Length 836
Episode 464 total samples 233258 train steps 6977
Length 1496
Episode 471 total samples 235770 train steps 7055
Length 588
Episode 478 total samples 239295 train steps 7165
Length 2122
Episode 484 total samples 241885 train steps 7246
Length 679
Episode 492 total samples 244925 train steps 7341
Length 1256
Episode 500 total samples 247461 train steps 7421
Length 277
Episode 506 total samples 250636 train steps 7520
Length 1465
Episode 515 total samples 253294 train steps 7603
Length 326
exploration 0.770580100008
Episode 522 total samples 255944 train steps 7686
Length 721
Episode 529 total samples 258446 train steps 7764
Length 1465
Episode 535 total samples 261455 train steps 7858
Length 1315
Episode 541 total samples 264653 train steps 7958
Length 1407
Episode 548 total samples 267443 train steps 8045
Length 584
Episode 556 total samples 270284 train steps 8134
Length 705
Episode 562 total samples 272846 train steps 8214
Length 411
Episode 568 total samples 275625 train steps 8301
Length 969
Episode 574 total samples 278319 train steps 8385
Length 1123
exploration 0.747988300008
Episode 582 total samples 281103 train steps 8472
Length 591
Episode 587 total samples 283825 train steps 8557
Length 1553
Episode 593 total samples 286388 train steps 8637
Length 442
Episode 601 total samples 290241 train steps 8758
Length 2785
Episode 604 total samples 292790 train steps 8837
Length 1273
Episode 610 total samples 295504 train steps 8922
Length 1687
Episode 616 total samples 298807 train steps 9025
Length 2166
Episode 624 total samples 301423 train steps 9107
Length 915
Episode 631 total samples 304723 train steps 9210
Length 1663
exploration 0.725338000009
Episode 641 total samples 307637 train steps 9301
Length 1136
Episode 651 total samples 310461 train steps 9389
Length 928
Episode 658 total samples 313182 train steps 9474
Length 768
Episode 665 total samples 315714 train steps 9554
Length 725
Episode 672 total samples 318678 train steps 9646
Length 1165
Episode 680 total samples 321438 train steps 9732
Length 1404
Episode 688 total samples 324744 train steps 9836
Length 2107
Episode 695 total samples 327351 train steps 9917
Length 555
Episode 702 total samples 330100 train steps 10003
Length 938
exploration 0.70221790001
Episode 706 total samples 332824 train steps 10088
Length 580
Episode 712 total samples 335414 train steps 10169
Length 743
Episode 718 total samples 338216 train steps 10257
Length 2003
Episode 724 total samples 340874 train steps 10340
Length 1050
Episode 730 total samples 343680 train steps 10427
Length 2784
Episode 739 total samples 346720 train steps 10522
Length 1206
Episode 744 total samples 349313 train steps 10604
Length 2054
Episode 749 total samples 352069 train steps 10690
Length 985
Episode 756 total samples 354952 train steps 10780
Length 1541
exploration 0.679352500011
Episode 764 total samples 358061 train steps 10877
Length 1287
Episode 774 total samples 360864 train steps 10964
Length 778
Episode 780 total samples 363957 train steps 11061
Length 1231
Episode 789 total samples 366509 train steps 11141
Length 579
Episode 794 total samples 369141 train steps 11223
Length 1271
Episode 801 total samples 372046 train steps 11314
Length 1031
Episode 807 total samples 374637 train steps 11395
Length 701
Episode 815 total samples 377453 train steps 11483
Length 650
Episode 822 total samples 380321 train steps 11573
Length 1235
exploration 0.656583400011
Episode 830 total samples 383129 train steps 11660
Length 756
Episode 838 total samples 385664 train steps 11739
Length 787
Episode 845 total samples 388590 train steps 11831
Length 994
Episode 850 total samples 391140 train steps 11911
Length 568
Episode 856 total samples 393965 train steps 11999
Length 1541
Episode 861 total samples 396997 train steps 12094
Length 1487
Episode 868 total samples 399839 train steps 12182
Length 732
Episode 876 total samples 402506 train steps 12266
Length 388
Episode 884 total samples 405019 train steps 12344
Length 1118
exploration 0.633803500012
Episode 890 total samples 407837 train steps 12432
Length 1473
Episode 896 total samples 410541 train steps 12517
Length 647
Episode 904 total samples 413228 train steps 12601
Length 735
Episode 914 total samples 415762 train steps 12680
Length 367
Episode 921 total samples 418354 train steps 12761
Length 921
Episode 929 total samples 420992 train steps 12843
Length 371
Episode 937 total samples 423511 train steps 12922
Length 614
Episode 943 total samples 426210 train steps 13007
Length 787
Episode 954 total samples 428813 train steps 13088
Length 344
Episode 963 total samples 431548 train steps 13173
Length 730
exploration 0.611262100013
Episode 967 total samples 434276 train steps 13259
Length 1080
Episode 974 total samples 436953 train steps 13342
Length 929
Episode 980 total samples 439906 train steps 13435
Length 1465
Episode 989 total samples 442627 train steps 13520
Length 1267
Episode 995 total samples 445464 train steps 13608
Length 1354
Episode 1002 total samples 448105 train steps 13691
Length 611
Episode 1009 total samples 451522 train steps 13798
Length 1900
Episode 1017 total samples 454552 train steps 13892
Length 1408
Episode 1022 total samples 457081 train steps 13971
Length 1061
exploration 0.588627100014
Episode 1031 total samples 459740 train steps 14054
Length 546
Episode 1039 total samples 462592 train steps 14143
Length 930
Episode 1048 total samples 465132 train steps 14223
Length 476
Episode 1055 total samples 467736 train steps 14304
Length 882
Episode 1065 total samples 470267 train steps 14383
Length 271
Episode 1074 total samples 472966 train steps 14468
Length 469
Episode 1084 total samples 475751 train steps 14555
Length 879
Episode 1091 total samples 478424 train steps 14638
Length 491
Episode 1099 total samples 481004 train steps 14719
Length 466
exploration 0.565834600014
Episode 1108 total samples 483510 train steps 14797
Length 147
Episode 1117 total samples 486041 train steps 14876
Length 314
Episode 1125 total samples 488638 train steps 14957
Length 442
Episode 1133 total samples 491495 train steps 15047
Length 835
Episode 1144 total samples 494270 train steps 15133
Length 1095
Episode 1152 total samples 496932 train steps 15217
Length 840
Episode 1159 total samples 499709 train steps 15303
Length 1048
Episode 1167 total samples 502230 train steps 15382
Length 547
Episode 1175 total samples 504976 train steps 15468
Length 679
Episode 1181 total samples 507610 train steps 15550
Length 709
exploration 0.543151000015
Episode 1190 total samples 510257 train steps 15633
Length 765
Episode 1202 total samples 513179 train steps 15724
Length 1211
Episode 1212 total samples 515706 train steps 15803
Length 599
Episode 1223 total samples 518303 train steps 15884
Length 395
Episode 1233 total samples 520813 train steps 15963
Length 418
Episode 1239 total samples 523405 train steps 16044
Length 1078
Episode 1247 total samples 525952 train steps 16123
Length 677
Episode 1255 total samples 528653 train steps 16208
Length 472
Episode 1264 total samples 531248 train steps 16289
Length 273
exploration 0.520606900016
Episode 1274 total samples 533761 train steps 16368
Length 415
Episode 1284 total samples 536309 train steps 16447
Length 505
Episode 1292 total samples 538950 train steps 16530
Length 508
Episode 1302 total samples 542145 train steps 16630
Length 1491
Episode 1312 total samples 544854 train steps 16714
Length 556
Episode 1318 total samples 547477 train steps 16796
Length 531
Episode 1325 total samples 550046 train steps 16876
Length 450
Episode 1333 total samples 552654 train steps 16958
Length 763
Episode 1343 total samples 555352 train steps 17042
Length 700
Episode 1349 total samples 558109 train steps 17128
Length 1143
exploration 0.497701900016
Episode 1357 total samples 561024 train steps 17219
Length 1090
Episode 1366 total samples 564106 train steps 17316
Length 1208
Episode 1376 total samples 566734 train steps 17398
Length 261
Episode 1385 total samples 569339 train steps 17479
Length 1354
Episode 1392 total samples 571922 train steps 17560
Length 251
Episode 1401 total samples 574484 train steps 17640
Length 579
Episode 1416 total samples 577139 train steps 17723
Length 506
Episode 1425 total samples 579698 train steps 17803
Length 1847
Episode 1436 total samples 582237 train steps 17882
Length 186
exploration 0.475171300016
Episode 1446 total samples 584855 train steps 17964
Length 849
Episode 1456 total samples 587423 train steps 18044
Length 279
Episode 1466 total samples 590045 train steps 18126
Length 878
Episode 1473 total samples 592634 train steps 18207
Length 798
Episode 1483 total samples 595196 train steps 18287
Length 414
Episode 1493 total samples 597810 train steps 18369
Length 230
Episode 1506 total samples 600365 train steps 18449
Length 229
Episode 1515 total samples 602964 train steps 18530
Length 411
Episode 1524 total samples 605635 train steps 18614
Length 400
Episode 1534 total samples 608315 train steps 18697
Length 1095
exploration 0.452516500015
Episode 1546 total samples 610966 train steps 18780
Length 621
Episode 1557 total samples 613585 train steps 18862
Length 263
Episode 1568 total samples 616149 train steps 18942
Length 745
Episode 1578 total samples 618760 train steps 19024
Length 689
Episode 1587 total samples 621345 train steps 19105
Length 286
Episode 1598 total samples 623920 train steps 19185
Length 891
Episode 1612 total samples 626722 train steps 19273
Length 726
Episode 1624 total samples 629495 train steps 19359
Length 572
Episode 1635 total samples 632080 train steps 19440
Length 627
exploration 0.429913900014
Episode 1646 total samples 634953 train steps 19530
Length 913
Episode 1656 total samples 637459 train steps 19608
Length 395
Episode 1668 total samples 640139 train steps 19692
Length 657
Episode 1675 total samples 642679 train steps 19771
Length 375
Episode 1687 total samples 645201 train steps 19850
Length 350
Episode 1700 total samples 647746 train steps 19930
Length 242
Episode 1716 total samples 650451 train steps 20014
Length 506
Episode 1726 total samples 653266 train steps 20102
Length 804
Episode 1737 total samples 655781 train steps 20181
Length 672
Episode 1749 total samples 658367 train steps 20261
Length 460
exploration 0.407230300014
Episode 1759 total samples 661096 train steps 20347
Length 613
Episode 1773 total samples 663683 train steps 20428
Length 257
Episode 1784 total samples 666315 train steps 20510
Length 294
Episode 1795 total samples 668911 train steps 20591
Length 315
Episode 1805 total samples 671474 train steps 20671
Length 451
Episode 1817 total samples 674122 train steps 20754
Length 663
Episode 1829 total samples 676678 train steps 20834
Length 369
Episode 1842 total samples 679258 train steps 20914
Length 331
Episode 1852 total samples 681847 train steps 20995
Length 577
exploration 0.384582700013
Episode 1864 total samples 684483 train steps 21078
Length 484
Episode 1876 total samples 687145 train steps 21161
Length 455
Episode 1885 total samples 689697 train steps 21241
Length 483
Episode 1900 total samples 692247 train steps 21320
Length 209
Episode 1915 total samples 694867 train steps 21402
Length 342
Episode 1928 total samples 697490 train steps 21484
Length 285
Episode 1942 total samples 700111 train steps 21566
Length 307
Episode 1957 total samples 702641 train steps 21645
Length 274
Episode 1971 total samples 705230 train steps 21726
Length 421
Episode 1987 total samples 707974 train steps 21812
Length 1006
exploration 0.361878400012
Episode 2004 total samples 710523 train steps 21891
Length 240
Episode 2018 total samples 713202 train steps 21975
Length 693
Episode 2031 total samples 715840 train steps 22057
Length 504
Episode 2044 total samples 718665 train steps 22146
Length 746
Episode 2061 total samples 721224 train steps 22226
Length 492
Episode 2074 total samples 723852 train steps 22308
Length 315
Episode 2084 total samples 726477 train steps 22390
Length 584
Episode 2096 total samples 729161 train steps 22474
Length 623
Episode 2111 total samples 732035 train steps 22564
Length 909
exploration 0.339071500012
Episode 2120 total samples 734753 train steps 22649
Length 457
Episode 2134 total samples 737352 train steps 22730
Length 323
Episode 2149 total samples 740092 train steps 22815
Length 480
Episode 2164 total samples 742619 train steps 22894
Length 296
Episode 2177 total samples 745273 train steps 22977
Length 432
Episode 2190 total samples 748275 train steps 23071
Length 1016
Episode 2203 total samples 750818 train steps 23151
Length 550
Episode 2215 total samples 753345 train steps 23230
Length 257
Episode 2228 total samples 756027 train steps 23313
Length 884
Episode 2244 total samples 758605 train steps 23394
Length 328
exploration 0.316512100011
Episode 2255 total samples 761120 train steps 23472
Length 155
Episode 2269 total samples 763793 train steps 23556
Length 1082
Episode 2282 total samples 766303 train steps 23634
Length 391
Episode 2297 total samples 768894 train steps 23715
Length 299
Episode 2312 total samples 771449 train steps 23795
Length 338
Episode 2328 total samples 773957 train steps 23874
Length 279
Episode 2343 total samples 776538 train steps 23954
Length 297
Episode 2359 total samples 779063 train steps 24033
Length 226
Episode 2374 total samples 781595 train steps 24112
Length 266
Episode 2392 total samples 784107 train steps 24191
Length 257
exploration 0.293924800011
Episode 2409 total samples 786781 train steps 24274
Length 412
Episode 2419 total samples 789435 train steps 24357
Length 634
Episode 2431 total samples 791992 train steps 24437
Length 237
Episode 2445 total samples 794750 train steps 24523
Length 649
Episode 2460 total samples 797280 train steps 24602
Length 315
Episode 2477 total samples 799824 train steps 24682
Length 368
Episode 2493 total samples 802473 train steps 24765
Length 368
Episode 2510 total samples 805078 train steps 24846
Length 523
Episode 2529 total samples 807826 train steps 24932
Length 502
exploration 0.27140770001
Episode 2547 total samples 810367 train steps 25011
Length 266
Episode 2566 total samples 812976 train steps 25093
Length 702
Episode 2583 total samples 815632 train steps 25176
Length 465
Episode 2600 total samples 818209 train steps 25257
Length 429
Episode 2617 total samples 820812 train steps 25338
Length 302
Episode 2635 total samples 823415 train steps 25419
Length 517
Episode 2652 total samples 825935 train steps 25498
Length 566
Episode 2668 total samples 828704 train steps 25584
Length 768
Episode 2684 total samples 831391 train steps 25668
Length 431
Episode 2700 total samples 833933 train steps 25748
Length 202
exploration 0.248805100009
Episode 2717 total samples 836609 train steps 25832
Length 561
Episode 2735 total samples 839157 train steps 25911
Length 311
Episode 2753 total samples 841684 train steps 25990
Length 392
Episode 2769 total samples 844262 train steps 26071
Length 345
Episode 2785 total samples 846814 train steps 26150
Length 543
Episode 2801 total samples 849409 train steps 26232
Length 261
Episode 2818 total samples 851918 train steps 26310
Length 393
Episode 2839 total samples 854536 train steps 26392
Length 252
Episode 2861 total samples 857149 train steps 26473
Length 275
Episode 2881 total samples 859874 train steps 26559
Length 572
exploration 0.226113400009
Episode 2900 total samples 862395 train steps 26637
Length 137
Episode 2916 total samples 865159 train steps 26724
Length 965
Episode 2937 total samples 867760 train steps 26805
Length 669
Episode 2951 total samples 870295 train steps 26884
Length 498
Episode 2966 total samples 872834 train steps 26964
Length 660
Episode 2986 total samples 875541 train steps 27048
Length 920
Episode 3008 total samples 878088 train steps 27128
Length 211
Episode 3027 total samples 880611 train steps 27207
Length 150
Episode 3045 total samples 883169 train steps 27287
Length 147
exploration 0.203602600009
Episode 3069 total samples 885882 train steps 27371
Length 467
Episode 3088 total samples 888686 train steps 27459
Length 1155
Episode 3105 total samples 891287 train steps 27540
Length 332
Episode 3123 total samples 893822 train steps 27619
Length 168
Episode 3143 total samples 896351 train steps 27698
Length 482
Episode 3163 total samples 898912 train steps 27778
Length 234
Episode 3190 total samples 901551 train steps 27861
Length 305
Episode 3207 total samples 904143 train steps 27942
Length 791
Episode 3224 total samples 906715 train steps 28022
Length 237
Episode 3246 total samples 909321 train steps 28104
Length 227
exploration 0.180980200009
Episode 3262 total samples 911852 train steps 28183
Length 121
Episode 3285 total samples 914446 train steps 28264
Length 216
Episode 3304 total samples 916954 train steps 28342
Length 137
Episode 3320 total samples 919491 train steps 28422
Length 112
Episode 3340 total samples 922000 train steps 28500
Length 688
Episode 3354 total samples 924555 train steps 28580
Length 841
Episode 3369 total samples 927283 train steps 28665
Length 700
Episode 3389 total samples 929819 train steps 28744
Length 264
Episode 3410 total samples 932349 train steps 28823
Length 379
Episode 3430 total samples 934851 train steps 28902
Length 429
exploration 0.158384800009
Episode 3447 total samples 937436 train steps 28982
Length 404
Episode 3475 total samples 940063 train steps 29064
Length 431
Episode 3493 total samples 942695 train steps 29147
Length 422
Episode 3513 total samples 945278 train steps 29227
Length 213
Episode 3540 total samples 947825 train steps 29307
Length 447
Episode 3559 total samples 950488 train steps 29390
Length 569
Episode 3584 total samples 953149 train steps 29473
Length 331
Episode 3608 total samples 955714 train steps 29554
Length 692
Episode 3628 total samples 958218 train steps 29632
Length 167
exploration 0.13567420001
Episode 3649 total samples 961024 train steps 29719
Length 826
Episode 3669 total samples 963555 train steps 29799
Length 118
Episode 3687 total samples 966139 train steps 29879
Length 348
Episode 3717 total samples 968690 train steps 29959
Length 149
Episode 3735 total samples 971309 train steps 30041
Length 387
Episode 3760 total samples 973917 train steps 30122
Length 431
Episode 3782 total samples 976485 train steps 30203
Length 259
Episode 3810 total samples 979052 train steps 30283
Length 155
Episode 3840 total samples 981593 train steps 30362
Length 143
Episode 3870 total samples 984133 train steps 30442
Length 113
exploration 0.11307700001
Episode 3894 total samples 986653 train steps 30520
Length 127
Episode 3921 total samples 989217 train steps 30601
Length 205
Episode 3944 total samples 991749 train steps 30680
Length 399
Episode 3966 total samples 994306 train steps 30760
Length 198
Episode 3992 total samples 996903 train steps 30841
Length 308
Episode 4013 total samples 999472 train steps 30921
Length 152
Episode 4045 total samples 1001992 train steps 31000
Length 126
Episode 4078 total samples 1004615 train steps 31082
Length 327
Episode 4102 total samples 1007140 train steps 31161
Length 110
Episode 4126 total samples 1009717 train steps 31241
Length 304
exploration 0.0999991000096
Episode 4153 total samples 1012265 train steps 31321
Length 128
Episode 4181 total samples 1014849 train steps 31402
Length 248
Episode 4205 total samples 1017547 train steps 31486
Length 458
Episode 4228 total samples 1020164 train steps 31568
Length 439
Episode 4247 total samples 1022878 train steps 31652
Length 462
Episode 4274 total samples 1025471 train steps 31733
Length 207
Episode 4302 total samples 1028051 train steps 31814
Length 508
Episode 4332 total samples 1030601 train steps 31894
Length 99
Episode 4358 total samples 1033255 train steps 31977
Length 317
exploration 0.0999991000096
Episode 4377 total samples 1035800 train steps 32056
Length 454
Episode 4400 total samples 1038309 train steps 32135
Length 384
Episode 4427 total samples 1040836 train steps 32214
Length 202
Episode 4457 total samples 1043380 train steps 32293
Length 147
Episode 4486 total samples 1045903 train steps 32372
Length 96
Episode 4511 total samples 1048448 train steps 32451
Length 120
Episode 4536 total samples 1051079 train steps 32534
Length 470
Episode 4561 total samples 1053600 train steps 32612
Length 85
Episode 4582 total samples 1056288 train steps 32696
Length 774
Episode 4603 total samples 1058915 train steps 32779
Length 310
exploration 0.0999991000096
Episode 4630 total samples 1061509 train steps 32860
Length 368
Episode 4656 total samples 1064027 train steps 32938
Length 293
Episode 4674 total samples 1066582 train steps 33018
Length 255
Episode 4691 total samples 1069138 train steps 33098
Length 345
Episode 4717 total samples 1071650 train steps 33177
Length 124
Episode 4750 total samples 1074363 train steps 33261
Length 438
Episode 4773 total samples 1076878 train steps 33340
Length 143
Episode 4806 total samples 1079426 train steps 33420
Length 144
Episode 4837 total samples 1081963 train steps 33499
Length 123
Episode 4863 total samples 1084499 train steps 33578
Length 186
exploration 0.0999991000096
Episode 4884 total samples 1087016 train steps 33657
Length 126
Episode 4913 total samples 1089561 train steps 33736
Length 205
Episode 4940 total samples 1092093 train steps 33815
Length 187
Episode 4969 total samples 1094617 train steps 33894
Length 90
Episode 4997 total samples 1097307 train steps 33978
Length 400
Episode 5028 total samples 1099897 train steps 34059
Length 326
Episode 5056 total samples 1102509 train steps 34141
Length 267
Episode 5087 total samples 1105180 train steps 34224
Length 463
Episode 5119 total samples 1107742 train steps 34304
Length 192
Episode 5152 total samples 1110295 train steps 34384
Length 196
exploration 0.0999991000096
Episode 5183 total samples 1112796 train steps 34462
Length 218
Episode 5209 total samples 1115336 train steps 34542
Length 90
Episode 5246 total samples 1117859 train steps 34621
Length 170
Episode 5274 total samples 1120414 train steps 34700
Length 110
Episode 5309 total samples 1122934 train steps 34779
Length 74
Episode 5349 total samples 1125697 train steps 34866
Length 637
Episode 5377 total samples 1128198 train steps 34944
Length 103
Episode 5406 total samples 1130710 train steps 35022
Length 237
Episode 5432 total samples 1133242 train steps 35101
Length 192
Episode 5463 total samples 1135767 train steps 35180
Length 97
exploration 0.0999991000096
Episode 5494 total samples 1138301 train steps 35259
Length 160
Episode 5527 total samples 1140893 train steps 35340
Length 201
Episode 5565 total samples 1143404 train steps 35419
Length 88
Episode 5594 total samples 1145925 train steps 35498
Length 140
Episode 5633 total samples 1148488 train steps 35578
Length 151
Episode 5669 total samples 1150999 train steps 35656
Length 88
Episode 5707 total samples 1153518 train steps 35735
Length 126
Episode 5738 total samples 1156099 train steps 35816
Length 170
Episode 5768 total samples 1158623 train steps 35894
Length 170
exploration 0.0999991000096
Episode 5805 total samples 1161203 train steps 35975
Length 290
Episode 5844 total samples 1163811 train steps 36057
Length 403
Episode 5880 total samples 1166337 train steps 36136
Length 121
Episode 5910 total samples 1168847 train steps 36214
Length 126
Episode 5945 total samples 1171397 train steps 36294
Length 278
Episode 5973 total samples 1174006 train steps 36375
Length 249
Episode 6005 total samples 1176574 train steps 36455
Length 143
Episode 6039 total samples 1179086 train steps 36534
Length 105
Episode 6079 total samples 1181663 train steps 36614
Length 173
Episode 6111 total samples 1184253 train steps 36695
Length 407
exploration 0.0999991000096
Episode 6150 total samples 1186821 train steps 36776
Length 375
Episode 6185 total samples 1189343 train steps 36854
Length 526
Episode 6221 total samples 1191875 train steps 36934
Length 162
Episode 6253 total samples 1194405 train steps 37013
Length 384
Episode 6287 total samples 1197000 train steps 37094
Length 261
Episode 6319 total samples 1199505 train steps 37172
Length 137
Episode 6354 total samples 1202023 train steps 37251
Length 238
Episode 6386 total samples 1204549 train steps 37330
Length 314
Episode 6424 total samples 1207098 train steps 37409
Length 101
Episode 6455 total samples 1209668 train steps 37490
Length 337
exploration 0.0999991000096
Episode 6489 total samples 1212224 train steps 37569
Length 132
Episode 6523 total samples 1214768 train steps 37649
Length 342
Episode 6562 total samples 1217292 train steps 37728
Length 223
Episode 6599 total samples 1219831 train steps 37807
Length 101
Episode 6632 total samples 1222383 train steps 37887
Length 122
Episode 6666 total samples 1224914 train steps 37966
Length 223
Episode 6702 total samples 1227448 train steps 38045
Length 138
Episode 6740 total samples 1229964 train steps 38124
Length 135
Episode 6772 total samples 1232486 train steps 38203
Length 128
Episode 6809 total samples 1235058 train steps 38283
Length 443
exploration 0.0999991000096
Episode 6843 total samples 1237609 train steps 38363
Length 143
Episode 6891 total samples 1240134 train steps 38442
Length 90
Episode 6933 total samples 1242704 train steps 38522
Length 189
Episode 6974 total samples 1245259 train steps 38602
Length 123
Episode 7011 total samples 1247791 train steps 38681
Length 148
Episode 7050 total samples 1250297 train steps 38759
Length 126
Episode 7090 total samples 1252806 train steps 38838
Length 186
Episode 7135 total samples 1255337 train steps 38917
Length 117
Episode 7176 total samples 1257929 train steps 38998
Length 351
Episode 7213 total samples 1260445 train steps 39076
Length 144
exploration 0.0999991000096
Episode 7246 total samples 1262951 train steps 39155
Length 91
Episode 7293 total samples 1265535 train steps 39235
Length 200
Episode 7333 total samples 1268059 train steps 39314
Length 103
Episode 7378 total samples 1270770 train steps 39399
Length 507
Episode 7417 total samples 1273289 train steps 39478
Length 127
Episode 7458 total samples 1275832 train steps 39557
Length 120
Episode 7501 total samples 1278429 train steps 39638
Length 246
Episode 7547 total samples 1281041 train steps 39720
Length 229
Episode 7588 total samples 1283648 train steps 39801
Length 231
exploration 0.0999991000096
Episode 7627 total samples 1286200 train steps 39881
Length 208
Episode 7667 total samples 1288847 train steps 39964
Length 417
Episode 7715 total samples 1291426 train steps 40045
Length 181
Episode 7758 total samples 1293984 train steps 40124
Length 216
Episode 7800 total samples 1296632 train steps 40207
Length 300
Episode 7846 total samples 1299151 train steps 40286
Length 81
Episode 7891 total samples 1301671 train steps 40365
Length 89
Episode 7927 total samples 1304187 train steps 40443
Length 99
Episode 7970 total samples 1306694 train steps 40522
Length 93
Episode 8013 total samples 1309257 train steps 40602
Length 155
exploration 0.0999991000096
Episode 8056 total samples 1311767 train steps 40680
Length 88
Episode 8097 total samples 1314313 train steps 40760
Length 108
Episode 8135 total samples 1316819 train steps 40838
Length 136
Episode 8180 total samples 1319320 train steps 40916
Length 76
Episode 8216 total samples 1321853 train steps 40995
Length 111
Episode 8259 total samples 1324469 train steps 41077
Length 264
Episode 8305 total samples 1327020 train steps 41157
Length 108
Episode 8348 total samples 1329524 train steps 41235
Length 117
Episode 8391 total samples 1332026 train steps 41313
Length 115
Episode 8437 total samples 1334564 train steps 41393
Length 82
exploration 0.0999991000096
Episode 8484 total samples 1337082 train steps 41471
Length 115
Episode 8525 total samples 1339612 train steps 41550
Length 123
Episode 8571 total samples 1342196 train steps 41631
Length 217
Episode 8615 total samples 1344757 train steps 41711
Length 150
Episode 8655 total samples 1347297 train steps 41791
Length 96
Episode 8703 total samples 1349829 train steps 41870
Length 87
Episode 8749 total samples 1352339 train steps 41948
Length 174
Episode 8795 total samples 1354876 train steps 42027
Length 93
Episode 8845 total samples 1357408 train steps 42106
Length 92
Episode 8887 total samples 1359912 train steps 42185
Length 99
exploration 0.0999991000096
Episode 8924 total samples 1362430 train steps 42263
Length 165
Episode 8969 total samples 1364978 train steps 42343
Length 108
Episode 9017 total samples 1367509 train steps 42422
Length 92
Episode 9066 total samples 1370046 train steps 42501
Length 85
Episode 9107 total samples 1372551 train steps 42580
Length 107
Episode 9156 total samples 1375079 train steps 42659
Length 171
Episode 9199 total samples 1377587 train steps 42737
Length 112
Episode 9248 total samples 1380146 train steps 42817
Length 182
Episode 9296 total samples 1382653 train steps 42895
Length 108
Episode 9343 total samples 1385187 train steps 42975
Length 90
exploration 0.0999991000096
Episode 9394 total samples 1387723 train steps 43054
Length 102
Episode 9445 total samples 1390265 train steps 43133
Length 197
Episode 9495 total samples 1392791 train steps 43212
Length 125
Episode 9542 total samples 1395304 train steps 43291
Length 99
Episode 9593 total samples 1397828 train steps 43370
Length 84
Episode 9645 total samples 1400364 train steps 43449
Length 105
Episode 9696 total samples 1402869 train steps 43527
Length 134
Episode 9742 total samples 1405385 train steps 43606
Length 134
Episode 9791 total samples 1407897 train steps 43684
Length 96
Episode 9843 total samples 1410428 train steps 43763
Length 62
exploration 0.0999991000096
Episode 9894 total samples 1412967 train steps 43843
Length 82
Episode 9944 total samples 1415509 train steps 43922
Length 103
Episode 9993 total samples 1418029 train steps 44001
Length 80
Episode 10043 total samples 1420568 train steps 44080
Length 119
Episode 10093 total samples 1423074 train steps 44159
Length 77
Episode 10147 total samples 1425578 train steps 44237
Length 107
Episode 10197 total samples 1428090 train steps 44315
Length 81
Episode 10252 total samples 1430623 train steps 44394
Length 103
Episode 10300 total samples 1433124 train steps 44473
Length 154
Episode 10351 total samples 1435632 train steps 44551
Length 89
exploration 0.0999991000096
Episode 10400 total samples 1438149 train steps 44630
Length 98
Episode 10444 total samples 1440654 train steps 44708
Length 112
Episode 10496 total samples 1443199 train steps 44787
Length 100
Episode 10549 total samples 1445761 train steps 44868
Length 132
Episode 10598 total samples 1448292 train steps 44947
Length 72
Episode 10648 total samples 1450814 train steps 45025
Length 82
Episode 10699 total samples 1453333 train steps 45104
Length 143
Episode 10752 total samples 1455846 train steps 45183
Length 94
Episode 10804 total samples 1458365 train steps 45261
Length 80
Episode 10857 total samples 1460878 train steps 45340
Length 113
exploration 0.0999991000096
Episode 10908 total samples 1463408 train steps 45419
Length 94
Episode 10956 total samples 1465956 train steps 45499
Length 203
Episode 11007 total samples 1468497 train steps 45578
Length 108
Episode 11058 total samples 1471003 train steps 45656
Length 118
Episode 11108 total samples 1473523 train steps 45735
Length 77
Episode 11162 total samples 1476066 train steps 45815
Length 100
Episode 11212 total samples 1478607 train steps 45894
Length 128
Episode 11260 total samples 1481115 train steps 45972
Length 169
Episode 11311 total samples 1483628 train steps 46051
Length 99
Episode 11365 total samples 1486155 train steps 46130
Length 78
exploration 0.0999991000096
Episode 11417 total samples 1488673 train steps 46209
Length 96
Episode 11468 total samples 1491197 train steps 46287
Length 101
Episode 11519 total samples 1493716 train steps 46366
Length 104
Episode 11567 total samples 1496235 train steps 46445
Length 99
Episode 11621 total samples 1498766 train steps 46524
Length 83
Episode 11673 total samples 1501281 train steps 46603
Length 86
Episode 11723 total samples 1503784 train steps 46681
Length 122
Episode 11776 total samples 1506295 train steps 46759
Length 129
Episode 11830 total samples 1508799 train steps 46837
Length 111
Episode 11882 total samples 1511315 train steps 46916
Length 91
exploration 0.0999991000096
Episode 11935 total samples 1513839 train steps 46995
Length 135
Episode 11986 total samples 1516349 train steps 47073
Length 111
Episode 12037 total samples 1518878 train steps 47152
Length 136
Episode 12090 total samples 1521388 train steps 47231
Length 83
Episode 12143 total samples 1523911 train steps 47310
Length 84
Episode 12196 total samples 1526420 train steps 47388
Length 140
Episode 12249 total samples 1528923 train steps 47466
Length 104
Episode 12299 total samples 1531435 train steps 47545
Length 92
Episode 12352 total samples 1533961 train steps 47624
Length 106
Episode 12404 total samples 1536489 train steps 47703
Length 101
exploration 0.0999991000096
Episode 12457 total samples 1538999 train steps 47781
Length 100
Episode 12511 total samples 1541503 train steps 47859
Length 91
Episode 12566 total samples 1544078 train steps 47940
Length 186
Episode 12617 total samples 1546596 train steps 48019
Length 104
Episode 12668 total samples 1549104 train steps 48097
Length 90
Episode 12718 total samples 1551645 train steps 48176
Length 83
Episode 12772 total samples 1554163 train steps 48255
Length 78
Episode 12826 total samples 1556712 train steps 48335
Length 104
Episode 12878 total samples 1559214 train steps 48413
Length 110
exploration 0.0999991000096
Episode 12933 total samples 1561752 train steps 48492
Length 91
Episode 12986 total samples 1564283 train steps 48571
Length 84
Episode 13038 total samples 1566812 train steps 48650
Length 90
Episode 13088 total samples 1569314 train steps 48729
Length 74
Episode 13139 total samples 1571845 train steps 48808
Length 94
Episode 13192 total samples 1574370 train steps 48887
Length 94
Episode 13245 total samples 1576899 train steps 48966
Length 94
Episode 13298 total samples 1579431 train steps 49045
Length 94
Episode 13352 total samples 1581963 train steps 49124
Length 77
Episode 13402 total samples 1584468 train steps 49202
Length 93
exploration 0.0999991000096
Episode 13454 total samples 1587015 train steps 49282
Length 99
Episode 13505 total samples 1589521 train steps 49360
Length 129
Episode 13555 total samples 1592022 train steps 49438
Length 80
Episode 13609 total samples 1594568 train steps 49518
Length 108
Episode 13662 total samples 1597102 train steps 49597
Length 110
Episode 13716 total samples 1599624 train steps 49676
Length 77
Episode 13770 total samples 1602154 train steps 49755
Length 99
Episode 13821 total samples 1604659 train steps 49833
Length 94
Episode 13871 total samples 1607175 train steps 49912
Length 93
Episode 13925 total samples 1609690 train steps 49990
Length 73
exploration 0.0999991000096
Episode 13978 total samples 1612212 train steps 50069
Length 92
Episode 14032 total samples 1614757 train steps 50149
Length 116
Episode 14084 total samples 1617278 train steps 50227
Length 72
Episode 14138 total samples 1619817 train steps 50307
Length 113
Episode 14190 total samples 1622330 train steps 50385
Length 136
Episode 14242 total samples 1624856 train steps 50464
Length 108
Episode 14297 total samples 1627367 train steps 50543
Length 94
Episode 14349 total samples 1629871 train steps 50621
Length 64
Episode 14404 total samples 1632414 train steps 50700
Length 92
Episode 14458 total samples 1634957 train steps 50780
Length 101
exploration 0.0999991000096
Episode 14513 total samples 1637472 train steps 50858
Length 62
Episode 14565 total samples 1639985 train steps 50937
Length 84
Episode 14616 total samples 1642552 train steps 51017
Length 168
Episode 14672 total samples 1645088 train steps 51096
Length 110
Episode 14726 total samples 1647607 train steps 51175
Length 110
Episode 14780 total samples 1650118 train steps 51254
Length 89
Episode 14833 total samples 1652657 train steps 51333
Length 113
Episode 14886 total samples 1655159 train steps 51411
Length 96
Episode 14940 total samples 1657669 train steps 51490
Length 82
Episode 14993 total samples 1660175 train steps 51568
Length 107
exploration 0.0999991000096
Episode 15046 total samples 1662709 train steps 51647
Length 100
Episode 15099 total samples 1665235 train steps 51726
Length 116
Episode 15153 total samples 1667743 train steps 51804
Length 100
Episode 15207 total samples 1670256 train steps 51883
Length 105
Episode 15261 total samples 1672775 train steps 51962
Length 91
Episode 15314 total samples 1675281 train steps 52040
Length 77
Episode 15365 total samples 1677785 train steps 52118
Length 81
Episode 15418 total samples 1680301 train steps 52197
Length 82
Episode 15472 total samples 1682838 train steps 52276
Length 87
Episode 15525 total samples 1685357 train steps 52355
Length 70
exploration 0.0999991000096
Episode 15578 total samples 1687859 train steps 52433
Length 93
Episode 15634 total samples 1690389 train steps 52512
Length 65
Episode 15690 total samples 1692941 train steps 52592
Length 104
Episode 15746 total samples 1695471 train steps 52671
Length 81
Episode 15803 total samples 1698008 train steps 52750
Length 76
Episode 15857 total samples 1700567 train steps 52830
Length 117
Episode 15907 total samples 1703073 train steps 52909
Length 84
Episode 15961 total samples 1705613 train steps 52988
Length 80
Episode 16016 total samples 1708145 train steps 53067
Length 81
Episode 16069 total samples 1710646 train steps 53145
Length 91
exploration 0.0999991000096
Episode 16123 total samples 1713169 train steps 53224
Length 82
Episode 16176 total samples 1715683 train steps 53303
Length 102
Episode 16231 total samples 1718224 train steps 53382
Length 87
Episode 16284 total samples 1720776 train steps 53462
Length 105
Episode 16338 total samples 1723299 train steps 53541
Length 90
Episode 16391 total samples 1725816 train steps 53619
Length 107
Episode 16444 total samples 1728336 train steps 53698
Length 86
Episode 16498 total samples 1730864 train steps 53777
Length 75
Episode 16551 total samples 1733378 train steps 53856
Length 121
Episode 16602 total samples 1735896 train steps 53934
Length 116
exploration 0.0999991000096
Episode 16654 total samples 1738414 train steps 54013
Length 77
Episode 16708 total samples 1740922 train steps 54091
Length 114
Episode 16762 total samples 1743445 train steps 54170
Length 82
Episode 16815 total samples 1745967 train steps 54249
Length 111
Episode 16869 total samples 1748473 train steps 54327
Length 79
Episode 16923 total samples 1751001 train steps 54406
Length 99
Episode 16976 total samples 1753510 train steps 54485
Length 123
Episode 17029 total samples 1756049 train steps 54564
Length 86
Episode 17083 total samples 1758591 train steps 54643
Length 106
Episode 17139 total samples 1761121 train steps 54723
Length 123
exploration 0.0999991000096
Episode 17195 total samples 1763625 train steps 54801
Length 81
Episode 17247 total samples 1766153 train steps 54880
Length 81
Episode 17301 total samples 1768658 train steps 54958
Length 99
Episode 17357 total samples 1771183 train steps 55037
Length 85
Episode 17412 total samples 1773720 train steps 55116
Length 92
Episode 17466 total samples 1776227 train steps 55195
Length 103
Episode 17521 total samples 1778744 train steps 55273
Length 85
Episode 17575 total samples 1781266 train steps 55352
Length 96
Episode 17629 total samples 1783806 train steps 55431
Length 99
Episode 17683 total samples 1786328 train steps 55510
Length 112
exploration 0.0999991000096
Episode 17739 total samples 1788842 train steps 55589
Length 88
Episode 17794 total samples 1791372 train steps 55668
Length 89
Episode 17846 total samples 1793887 train steps 55746
Length 77
Episode 17901 total samples 1796413 train steps 55825
Length 115
Episode 17954 total samples 1798922 train steps 55904
Length 80
Episode 18010 total samples 1801495 train steps 55984
Length 145
Episode 18066 total samples 1804020 train steps 56063
Length 107
Episode 18123 total samples 1806548 train steps 56142
Length 85
Episode 18178 total samples 1809063 train steps 56221
Length 99
Episode 18232 total samples 1811593 train steps 56300
Length 131
exploration 0.0999991000096
Episode 18286 total samples 1814117 train steps 56379
Length 112
Episode 18340 total samples 1816680 train steps 56459
Length 131
Episode 18392 total samples 1819219 train steps 56538
Length 103
Episode 18446 total samples 1821742 train steps 56617
Length 94
Episode 18500 total samples 1824244 train steps 56695
Length 82
Episode 18549 total samples 1826789 train steps 56775
Length 105
Episode 18604 total samples 1829352 train steps 56855
Length 133
Episode 18657 total samples 1831853 train steps 56933
Length 94
Episode 18712 total samples 1834363 train steps 57011
Length 72
exploration 0.0999991000096
Episode 18767 total samples 1836871 train steps 57090
Length 84
Episode 18820 total samples 1839410 train steps 57169
Length 90
Episode 18876 total samples 1841925 train steps 57248
Length 80
Episode 18933 total samples 1844428 train steps 57326
Length 101
Episode 18988 total samples 1846929 train steps 57404
Length 108
Episode 19042 total samples 1849472 train steps 57483
Length 87
Episode 19095 total samples 1852009 train steps 57563
Length 103
Episode 19151 total samples 1854511 train steps 57641
Length 71
Episode 19209 total samples 1857052 train steps 57720
Length 90
Episode 19262 total samples 1859563 train steps 57799
Length 64
exploration 0.0999991000096
Episode 19318 total samples 1862067 train steps 57877
Length 82
Episode 19374 total samples 1864571 train steps 57955
Length 79
Episode 19428 total samples 1867121 train steps 58035
Length 156
Episode 19483 total samples 1869630 train steps 58113
Length 101
Episode 19540 total samples 1872142 train steps 58192
Length 97
Episode 19596 total samples 1874676 train steps 58271
Length 129
Episode 19650 total samples 1877205 train steps 58350
Length 82
Episode 19706 total samples 1879719 train steps 58429
Length 103
Episode 19761 total samples 1882221 train steps 58507
Length 65
Episode 19818 total samples 1884747 train steps 58586
Length 106
exploration 0.0999991000096
Episode 19873 total samples 1887263 train steps 58664
Length 114
Episode 19928 total samples 1889781 train steps 58743
Length 80
Episode 19985 total samples 1892315 train steps 58822
Length 89
Episode 20041 total samples 1894893 train steps 58903
Length 180
Episode 20095 total samples 1897425 train steps 58982
Length 116
Episode 20148 total samples 1899966 train steps 59061
Length 92
Episode 20203 total samples 1902471 train steps 59140
Length 84
Episode 20260 total samples 1904997 train steps 59219
Length 97
Episode 20315 total samples 1907507 train steps 59297
Length 89
Episode 20371 total samples 1910055 train steps 59377
Length 95
exploration 0.0999991000096
Episode 20424 total samples 1912559 train steps 59455
Length 93
Episode 20477 total samples 1915106 train steps 59535
Length 93
Episode 20532 total samples 1917624 train steps 59613
Length 70
Episode 20589 total samples 1920169 train steps 59693
Length 103
Episode 20645 total samples 1922694 train steps 59772
Length 72
Episode 20699 total samples 1925197 train steps 59850
Length 74
Episode 20755 total samples 1927704 train steps 59928
Length 96
Episode 20809 total samples 1930234 train steps 60007
Length 104
Episode 20863 total samples 1932744 train steps 60086
Length 75
Episode 20917 total samples 1935250 train steps 60164
Length 87
exploration 0.0999991000096
Episode 20973 total samples 1937760 train steps 60242
Length 85
Episode 21028 total samples 1940279 train steps 60321
Length 84
Episode 21082 total samples 1942810 train steps 60400
Length 106
Episode 21138 total samples 1945323 train steps 60479
Length 98
Episode 21194 total samples 1947871 train steps 60558
Length 105
Episode 21246 total samples 1950388 train steps 60637
Length 149
Episode 21303 total samples 1952906 train steps 60716
Length 105
Episode 21360 total samples 1955423 train steps 60794
Length 86
Episode 21415 total samples 1957963 train steps 60874
Length 84
Episode 21469 total samples 1960466 train steps 60952
Length 127
exploration 0.0999991000096
Episode 21526 total samples 1962998 train steps 61031
Length 93
Episode 21581 total samples 1965523 train steps 61110
Length 111
Episode 21636 total samples 1968055 train steps 61189
Length 92
Episode 21693 total samples 1970571 train steps 61268
Length 73
Episode 21747 total samples 1973079 train steps 61346
Length 81
Episode 21803 total samples 1975619 train steps 61426
Length 89
Episode 21859 total samples 1978148 train steps 61505
Length 104
Episode 21916 total samples 1980655 train steps 61583
Length 101
Episode 21973 total samples 1983176 train steps 61662
Length 66
Episode 22028 total samples 1985720 train steps 61741
Length 101
exploration 0.0999991000096
Episode 22085 total samples 1988221 train steps 61819
Length 76
Episode 22142 total samples 1990759 train steps 61899
Length 75
Episode 22199 total samples 1993268 train steps 61977
Length 101
Episode 22252 total samples 1995802 train steps 62056
Length 74
Episode 22309 total samples 1998338 train steps 62136
Length 78
Episode 22363 total samples 2000883 train steps 62215
Length 103
Episode 22416 total samples 2003389 train steps 62293
Length 76
Episode 22471 total samples 2005924 train steps 62373
Length 80
Episode 22527 total samples 2008437 train steps 62451
Length 117
Episode 22583 total samples 2010943 train steps 62529
Length 101
exploration 0.0999991000096
Episode 22638 total samples 2013444 train steps 62608
Length 88
Episode 22694 total samples 2015985 train steps 62687
Length 140
Episode 22752 total samples 2018533 train steps 62767
Length 108
Episode 22809 total samples 2021064 train steps 62846
Length 84
Episode 22862 total samples 2023591 train steps 62925
Length 71
Episode 22918 total samples 2026117 train steps 63004
Length 85
Episode 22973 total samples 2028650 train steps 63083
Length 81
Episode 23031 total samples 2031182 train steps 63162
Length 109
Episode 23086 total samples 2033708 train steps 63241
Length 115
Episode 23140 total samples 2036211 train steps 63319
Length 96
exploration 0.0999991000096
Episode 23195 total samples 2038712 train steps 63397
Length 95
Episode 23252 total samples 2041219 train steps 63476
Length 75
Episode 23308 total samples 2043756 train steps 63555
Length 141
Episode 23364 total samples 2046258 train steps 63633
Length 95
Episode 23420 total samples 2048766 train steps 63711
Length 74
Episode 23476 total samples 2051267 train steps 63790
Length 131
Episode 23534 total samples 2053810 train steps 63869
Length 111
Episode 23588 total samples 2056337 train steps 63948
Length 75
Episode 23643 total samples 2058859 train steps 64027
Length 105
Episode 23696 total samples 2061370 train steps 64105
Length 88
exploration 0.0999991000096
Episode 23751 total samples 2063876 train steps 64184
Length 83
Episode 23809 total samples 2066411 train steps 64263
Length 129
Episode 23865 total samples 2068942 train steps 64342
Length 89
Episode 23922 total samples 2071450 train steps 64420
Length 80
Episode 23978 total samples 2073965 train steps 64499
Length 91
Episode 24032 total samples 2076479 train steps 64577
Length 62
Episode 24088 total samples 2079001 train steps 64656
Length 80
Episode 24143 total samples 2081538 train steps 64736
Length 105
Episode 24200 total samples 2084074 train steps 64815
Length 105
Episode 24256 total samples 2086590 train steps 64893
Length 111
exploration 0.0999991000096
Episode 24312 total samples 2089099 train steps 64972
Length 81
Episode 24370 total samples 2091627 train steps 65051
Length 87
Episode 24426 total samples 2094146 train steps 65130
Length 114
Episode 24483 total samples 2096672 train steps 65208
Length 82
Episode 24540 total samples 2099203 train steps 65288
Length 100
Episode 24595 total samples 2101706 train steps 65366
Length 105
Episode 24649 total samples 2104215 train steps 65444
Length 72
Episode 24700 total samples 2106724 train steps 65523
Length 96
Episode 24756 total samples 2109232 train steps 65601
Length 76
Episode 24813 total samples 2111742 train steps 65679
Length 102
exploration 0.0999991000096
Episode 24870 total samples 2114284 train steps 65759
Length 85
Episode 24925 total samples 2116790 train steps 65837
Length 103
Episode 24981 total samples 2119311 train steps 65916
Length 104
Episode 25038 total samples 2121848 train steps 65995
Length 110
Episode 25097 total samples 2124384 train steps 66074
Length 85
Episode 25154 total samples 2126921 train steps 66154
Length 115
Episode 25211 total samples 2129431 train steps 66232
Length 92
Episode 25269 total samples 2131958 train steps 66311
Length 109
Episode 25326 total samples 2134502 train steps 66391
Length 90
Episode 25384 total samples 2137034 train steps 66470
Length 82
exploration 0.0999991000096
Episode 25437 total samples 2139553 train steps 66549
Length 88
Episode 25493 total samples 2142072 train steps 66627
Length 63
Episode 25548 total samples 2144594 train steps 66706
Length 110
Episode 25604 total samples 2147100 train steps 66784
Length 101
Episode 25661 total samples 2149628 train steps 66863
Length 87
Episode 25718 total samples 2152160 train steps 66942
Length 79
Episode 25773 total samples 2154683 train steps 67021
Length 81
Episode 25831 total samples 2157203 train steps 67100
Length 84
Episode 25889 total samples 2159736 train steps 67179
Length 80
exploration 0.0999991000096
Episode 25947 total samples 2162285 train steps 67259
Length 97
Episode 26004 total samples 2164790 train steps 67337
Length 105
Episode 26062 total samples 2167310 train steps 67416
Length 87
Episode 26120 total samples 2169846 train steps 67495
Length 78
Episode 26177 total samples 2172395 train steps 67575
Length 117
Episode 26237 total samples 2174936 train steps 67654
Length 87
Episode 26296 total samples 2177439 train steps 67732
Length 112
Episode 26352 total samples 2179995 train steps 67812
Length 163
Episode 26410 total samples 2182511 train steps 67891
Length 83
Episode 26467 total samples 2185061 train steps 67971
Length 105
exploration 0.0999991000096
Episode 26523 total samples 2187601 train steps 68050
Length 92
Episode 26579 total samples 2190115 train steps 68129
Length 87
Episode 26636 total samples 2192656 train steps 68208
Length 96
Episode 26693 total samples 2195197 train steps 68287
Length 116
Episode 26748 total samples 2197730 train steps 68367
Length 130
Episode 26804 total samples 2200264 train steps 68446
Length 88
Episode 26859 total samples 2202801 train steps 68525
Length 91
Episode 26916 total samples 2205320 train steps 68604
Length 86
Episode 26972 total samples 2207854 train steps 68683
Length 93
Episode 27029 total samples 2210411 train steps 68763
Length 120
exploration 0.0999991000096
Episode 27087 total samples 2212942 train steps 68842
Length 71
Episode 27140 total samples 2215481 train steps 68921
Length 118
Episode 27197 total samples 2218029 train steps 69001
Length 95
Episode 27251 total samples 2220551 train steps 69080
Length 88
Episode 27306 total samples 2223061 train steps 69158
Length 101
Episode 27358 total samples 2225586 train steps 69237
Length 99
Episode 27413 total samples 2228098 train steps 69316
Length 121
Episode 27470 total samples 2230618 train steps 69394
Length 65
Episode 27528 total samples 2233163 train steps 69474
Length 89
Episode 27584 total samples 2235666 train steps 69552
Length 85
exploration 0.0999991000096
Episode 27638 total samples 2238170 train steps 69630
Length 96
Episode 27696 total samples 2240673 train steps 69709
Length 84
Episode 27753 total samples 2243189 train steps 69787
Length 72
Episode 27809 total samples 2245712 train steps 69866
Length 86
Episode 27867 total samples 2248237 train steps 69945
Length 72
Episode 27923 total samples 2250773 train steps 70024
Length 102
Episode 27977 total samples 2253282 train steps 70103
Length 82
Episode 28035 total samples 2255794 train steps 70181
Length 86
Episode 28093 total samples 2258316 train steps 70260
Length 93
Episode 28149 total samples 2260825 train steps 70338
Length 88
exploration 0.0999991000096
Episode 28206 total samples 2263365 train steps 70418
Length 89
Episode 28262 total samples 2265876 train steps 70496
Length 99
Episode 28319 total samples 2268391 train steps 70575
Length 65
Episode 28375 total samples 2270898 train steps 70653
Length 103
Episode 28431 total samples 2273414 train steps 70732
Length 85
Episode 28488 total samples 2275940 train steps 70811
Length 72
Episode 28543 total samples 2278491 train steps 70890
Length 102
Episode 28597 total samples 2281009 train steps 70969
Length 84
Episode 28656 total samples 2283545 train steps 71048
Length 79
Episode 28713 total samples 2286059 train steps 71127
Length 96
exploration 0.0999991000096
Episode 28771 total samples 2288569 train steps 71205
Length 87
Episode 28826 total samples 2291094 train steps 71284
Length 76
Episode 28886 total samples 2293623 train steps 71363
Length 64
Episode 28944 total samples 2296147 train steps 71442
Length 87
Episode 28999 total samples 2298686 train steps 71521
Length 78
Episode 29058 total samples 2301190 train steps 71600
Length 75
Episode 29115 total samples 2303701 train steps 71678
Length 89
Episode 29172 total samples 2306216 train steps 71757
Length 123
Episode 29228 total samples 2308720 train steps 71835
Length 111
Episode 29285 total samples 2311250 train steps 71914
Length 74
exploration 0.0999991000096
Episode 29339 total samples 2313762 train steps 71993
Length 102
Episode 29395 total samples 2316297 train steps 72072
Length 89
Episode 29453 total samples 2318814 train steps 72150
Length 104
Episode 29509 total samples 2321347 train steps 72230
Length 70
Episode 29565 total samples 2323858 train steps 72308
Length 72
Episode 29621 total samples 2326370 train steps 72387
Length 77
Episode 29676 total samples 2328913 train steps 72466
Length 125
Episode 29732 total samples 2331430 train steps 72545
Length 74
Episode 29788 total samples 2333953 train steps 72624
Length 87
Episode 29845 total samples 2336485 train steps 72703
Length 78
exploration 0.0999991000096
Episode 29903 total samples 2339019 train steps 72782
Length 81
Episode 29960 total samples 2341556 train steps 72861
Length 82
Episode 30019 total samples 2344059 train steps 72939
Length 89
Episode 30077 total samples 2346601 train steps 73019
Length 97
Episode 30133 total samples 2349142 train steps 73098
Length 83
Episode 30190 total samples 2351671 train steps 73177
Length 107
Episode 30247 total samples 2354206 train steps 73256
Length 86
Episode 30304 total samples 2356737 train steps 73336
Length 105
Episode 30361 total samples 2359254 train steps 73414
Length 75
Episode 30421 total samples 2361806 train steps 73494
Length 107
exploration 0.0999991000096
Episode 30476 total samples 2364333 train steps 73573
Length 81
Episode 30533 total samples 2366847 train steps 73651
Length 120
Episode 30589 total samples 2369357 train steps 73730
Length 63
Episode 30644 total samples 2371884 train steps 73809
Length 91
Episode 30698 total samples 2374451 train steps 73889
Length 198
Episode 30755 total samples 2377006 train steps 73969
Length 113
Episode 30810 total samples 2379526 train steps 74048
Length 82
Episode 30867 total samples 2382037 train steps 74126
Length 95
Episode 30923 total samples 2384542 train steps 74204
Length 111
Episode 30981 total samples 2387076 train steps 74284
Length 105
exploration 0.0999991000096
Episode 31040 total samples 2389593 train steps 74362
Length 102
Episode 31098 total samples 2392123 train steps 74441
Length 70
Episode 31151 total samples 2394631 train steps 74520
Length 90
Episode 31210 total samples 2397151 train steps 74598
Length 87
Episode 31267 total samples 2399664 train steps 74677
Length 78
Episode 31323 total samples 2402181 train steps 74756
Length 77
Episode 31378 total samples 2404698 train steps 74834
Length 70
Episode 31435 total samples 2407213 train steps 74913
Length 89
Episode 31491 total samples 2409741 train steps 74992
Length 89
Episode 31549 total samples 2412271 train steps 75071
Length 101
exploration 0.0999991000096
Episode 31606 total samples 2414814 train steps 75150
Length 98
Episode 31665 total samples 2417345 train steps 75230
Length 101
Episode 31720 total samples 2419851 train steps 75308
Length 92
Episode 31775 total samples 2422355 train steps 75386
Length 83
Episode 31833 total samples 2424874 train steps 75465
Length 72
Episode 31890 total samples 2427394 train steps 75544
Length 75
Episode 31948 total samples 2429931 train steps 75623
Length 83
Episode 32005 total samples 2432456 train steps 75702
Length 81
Episode 32062 total samples 2434976 train steps 75780
Length 78
exploration 0.0999991000096
Episode 32118 total samples 2437489 train steps 75859
Length 86
Episode 32173 total samples 2440017 train steps 75938
Length 133
Episode 32230 total samples 2442561 train steps 76018
Length 110
Episode 32284 total samples 2445066 train steps 76096
Length 89
Episode 32342 total samples 2447586 train steps 76175
Length 65
Episode 32396 total samples 2450097 train steps 76253
Length 114
Episode 32453 total samples 2452617 train steps 76332
Length 107
Episode 32508 total samples 2455139 train steps 76411
Length 99
Episode 32565 total samples 2457650 train steps 76489
Length 89
Episode 32622 total samples 2460159 train steps 76567
Length 76
exploration 0.0999991000096
Episode 32678 total samples 2462687 train steps 76646
Length 65
Episode 32735 total samples 2465198 train steps 76725
Length 75
Episode 32791 total samples 2467709 train steps 76803
Length 84
Episode 32851 total samples 2470229 train steps 76882
Length 63
Episode 32909 total samples 2472748 train steps 76961
Length 88
Episode 32970 total samples 2475269 train steps 77040
Length 87
Episode 33025 total samples 2477773 train steps 77118
Length 106
Episode 33082 total samples 2480278 train steps 77196
Length 93
Episode 33141 total samples 2482800 train steps 77275
Length 94
Episode 33195 total samples 2485320 train steps 77354
Length 78
exploration 0.0999991000096
Episode 33250 total samples 2487826 train steps 77432
Length 81
Episode 33306 total samples 2490335 train steps 77510
Length 77
Episode 33360 total samples 2492859 train steps 77589
Length 101
Episode 33417 total samples 2495373 train steps 77668
Length 97
Episode 33474 total samples 2497878 train steps 77746
Length 78
Episode 33532 total samples 2500428 train steps 77826
Length 103
Episode 33588 total samples 2502960 train steps 77905
Length 71
Episode 33646 total samples 2505475 train steps 77984
Length 118
Episode 33700 total samples 2507998 train steps 78062
Length 80
Episode 33757 total samples 2510526 train steps 78141
Length 65
exploration 0.0999991000096
Episode 33811 total samples 2513027 train steps 78220
Length 94
Episode 33867 total samples 2515576 train steps 78299
Length 107
Episode 33926 total samples 2518088 train steps 78378
Length 79
Episode 33983 total samples 2520630 train steps 78457
Length 112
Episode 34042 total samples 2523138 train steps 78536
Length 95
Episode 34099 total samples 2525641 train steps 78614
Length 108
Episode 34160 total samples 2528157 train steps 78692
Length 94
Episode 34217 total samples 2530668 train steps 78771
Length 75
Episode 34270 total samples 2533216 train steps 78850
Length 125
Episode 34329 total samples 2535721 train steps 78929
Length 65
exploration 0.0999991000096
Episode 34386 total samples 2538233 train steps 79007
Length 89
Episode 34444 total samples 2540774 train steps 79087
Length 98
Episode 34500 total samples 2543279 train steps 79165
Length 71
Episode 34557 total samples 2545808 train steps 79244
Length 95
Episode 34615 total samples 2548318 train steps 79322
Length 71
Episode 34670 total samples 2550855 train steps 79402
Length 75
Episode 34726 total samples 2553363 train steps 79480
Length 103
Episode 34780 total samples 2555883 train steps 79559
Length 145
Episode 34838 total samples 2558419 train steps 79638
Length 72
Episode 34896 total samples 2560934 train steps 79717
Length 84
exploration 0.0999991000096
Episode 34955 total samples 2563452 train steps 79795
Length 87
Episode 35012 total samples 2565961 train steps 79874
Length 104
Episode 35070 total samples 2568512 train steps 79953
Length 107
Episode 35129 total samples 2571050 train steps 80033
Length 87
Episode 35186 total samples 2573565 train steps 80111
Length 63
Episode 35244 total samples 2576102 train steps 80191
Length 81
Episode 35303 total samples 2578630 train steps 80270
Length 82
Episode 35360 total samples 2581162 train steps 80349
Length 83
Episode 35418 total samples 2583675 train steps 80427
Length 95
Episode 35475 total samples 2586204 train steps 80506
Length 99
exploration 0.0999991000096
Episode 35529 total samples 2588726 train steps 80585
Length 70
Episode 35588 total samples 2591279 train steps 80665
Length 108
Episode 35647 total samples 2593838 train steps 80745
Length 132
Episode 35703 total samples 2596349 train steps 80823
Length 109
Episode 35760 total samples 2598868 train steps 80902
Length 87
Episode 35817 total samples 2601416 train steps 80982
Length 95
Episode 35873 total samples 2603948 train steps 81061
Length 100
Episode 35928 total samples 2606480 train steps 81140
Length 95
Episode 35983 total samples 2608992 train steps 81218
Length 78
Episode 36041 total samples 2611523 train steps 81298
Length 63
exploration 0.0999991000096
Episode 36097 total samples 2614052 train steps 81377
Length 101
Episode 36155 total samples 2616586 train steps 81456
Length 73
Episode 36213 total samples 2619092 train steps 81534
Length 91
Episode 36273 total samples 2621609 train steps 81613
Length 76
Episode 36330 total samples 2624140 train steps 81692
Length 83
Episode 36385 total samples 2626648 train steps 81770
Length 82
Episode 36443 total samples 2629179 train steps 81849
Length 91
Episode 36501 total samples 2631711 train steps 81928
Length 72
Episode 36558 total samples 2634265 train steps 82008
Length 116
Episode 36615 total samples 2636807 train steps 82088
Length 96
exploration 0.0999991000096
Episode 36672 total samples 2639320 train steps 82166
Length 65
Episode 36728 total samples 2641853 train steps 82245
Length 70
Episode 36787 total samples 2644386 train steps 82325
Length 83
Episode 36841 total samples 2646918 train steps 82404
Length 99
Episode 36899 total samples 2649435 train steps 82482
Length 107
Episode 36957 total samples 2651969 train steps 82562
Length 79
Episode 37010 total samples 2654485 train steps 82640
Length 94
Episode 37065 total samples 2656993 train steps 82719
Length 94
Episode 37121 total samples 2659532 train steps 82798
Length 88
Episode 37177 total samples 2662069 train steps 82877
Length 100
exploration 0.0999991000096
Episode 37236 total samples 2664570 train steps 82955
Length 85
Episode 37292 total samples 2667080 train steps 83034
Length 65
Episode 37349 total samples 2669611 train steps 83113
Length 75
Episode 37406 total samples 2672115 train steps 83191
Length 91
Episode 37462 total samples 2674624 train steps 83269
Length 101
Episode 37522 total samples 2677127 train steps 83348
Length 70
Episode 37580 total samples 2679662 train steps 83427
Length 73
Episode 37637 total samples 2682194 train steps 83506
Length 81
Episode 37697 total samples 2684711 train steps 83585
Length 105
Episode 37755 total samples 2687219 train steps 83663
Length 100
exploration 0.0999991000096
Episode 37812 total samples 2689745 train steps 83742
Length 107
Episode 37869 total samples 2692271 train steps 83821
Length 72
Episode 37926 total samples 2694804 train steps 83900
Length 105
Episode 37982 total samples 2697322 train steps 83979
Length 84
Episode 38037 total samples 2699834 train steps 84057
Length 85
Episode 38093 total samples 2702366 train steps 84136
Length 94
Episode 38152 total samples 2704892 train steps 84215
Length 73
Episode 38213 total samples 2707408 train steps 84294
Length 82
Episode 38268 total samples 2709950 train steps 84373
Length 88
Episode 38322 total samples 2712454 train steps 84452
Length 104
exploration 0.0999991000096
Episode 38383 total samples 2714975 train steps 84530
Length 85
Episode 38441 total samples 2717497 train steps 84609
Length 108
Episode 38499 total samples 2720016 train steps 84688
Length 80
Episode 38554 total samples 2722542 train steps 84767
Length 77
Episode 38612 total samples 2725069 train steps 84846
Length 102
Episode 38668 total samples 2727603 train steps 84925
Length 79
Episode 38728 total samples 2730156 train steps 85005
Length 114
Episode 38784 total samples 2732667 train steps 85083
Length 102
Episode 38841 total samples 2735172 train steps 85162
Length 84
exploration 0.0999991000096
Episode 38899 total samples 2737719 train steps 85241
Length 103
Episode 38955 total samples 2740261 train steps 85321
Length 100
Episode 39014 total samples 2742787 train steps 85400
Length 162
Episode 39070 total samples 2745306 train steps 85478
Length 93
Episode 39127 total samples 2747825 train steps 85557
Length 95
Episode 39182 total samples 2750353 train steps 85636
Length 95
Episode 39238 total samples 2752900 train steps 85716
Length 122
Episode 39296 total samples 2755428 train steps 85795
Length 73
Episode 39352 total samples 2757977 train steps 85874
Length 101
Episode 39409 total samples 2760481 train steps 85953
Length 83
exploration 0.0999991000096
Episode 39467 total samples 2763005 train steps 86031
Length 73
Episode 39525 total samples 2765516 train steps 86110
Length 92
Episode 39580 total samples 2768026 train steps 86188
Length 99
Episode 39636 total samples 2770558 train steps 86267
Length 87
Episode 39694 total samples 2773089 train steps 86347
Length 141
Episode 39749 total samples 2775619 train steps 86426
Length 133
Episode 39806 total samples 2778120 train steps 86504
Length 86
Episode 39865 total samples 2780647 train steps 86583
Length 78
Episode 39924 total samples 2783176 train steps 86662
Length 79
Episode 39981 total samples 2785682 train steps 86740
Length 73
exploration 0.0999991000096
Episode 40039 total samples 2788208 train steps 86819
Length 71
Episode 40096 total samples 2790727 train steps 86898
Length 73
Episode 40154 total samples 2793275 train steps 86977
Length 163
Episode 40213 total samples 2795790 train steps 87056
Length 97
Episode 40269 total samples 2798295 train steps 87134
Length 62
Episode 40327 total samples 2800845 train steps 87214
Length 99
Episode 40382 total samples 2803372 train steps 87293
Length 81
Episode 40440 total samples 2805874 train steps 87371
Length 62
Episode 40495 total samples 2808386 train steps 87450
Length 88
Episode 40551 total samples 2810900 train steps 87528
Length 96
exploration 0.0999991000096
Episode 40606 total samples 2813401 train steps 87606
Length 93
Episode 40666 total samples 2815921 train steps 87685
Length 94
Episode 40722 total samples 2818467 train steps 87765
Length 112
Episode 40776 total samples 2820990 train steps 87843
Length 87
Episode 40835 total samples 2823534 train steps 87923
Length 97
Episode 40893 total samples 2826070 train steps 88002
Length 87
Episode 40949 total samples 2828600 train steps 88081
Length 87
Episode 41009 total samples 2831120 train steps 88160
Length 87
Episode 41067 total samples 2833636 train steps 88239
Length 109
Episode 41126 total samples 2836165 train steps 88318
Length 78
exploration 0.0999991000096
Episode 41185 total samples 2838710 train steps 88397
Length 99
Episode 41241 total samples 2841212 train steps 88475
Length 62
Episode 41300 total samples 2843762 train steps 88555
Length 138
Episode 41359 total samples 2846272 train steps 88633
Length 92
Episode 41416 total samples 2848793 train steps 88712
Length 109
Episode 41474 total samples 2851329 train steps 88792
Length 93
Episode 41533 total samples 2853834 train steps 88870
Length 104
Episode 41590 total samples 2856349 train steps 88948
Length 137
Episode 41645 total samples 2858875 train steps 89027
Length 128
Episode 41704 total samples 2861423 train steps 89107
Length 98
exploration 0.0999991000096
Episode 41762 total samples 2863953 train steps 89186
Length 62
Episode 41817 total samples 2866479 train steps 89265
Length 97
Episode 41873 total samples 2869017 train steps 89344
Length 77
Episode 41929 total samples 2871534 train steps 89423
Length 159
Episode 41987 total samples 2874063 train steps 89502
Length 95
Episode 42046 total samples 2876585 train steps 89581
Length 90
Episode 42103 total samples 2879090 train steps 89659
Length 70
Episode 42160 total samples 2881592 train steps 89737
Length 73
Episode 42215 total samples 2884134 train steps 89817
Length 84
Episode 42270 total samples 2886649 train steps 89895
Length 73
exploration 0.0999991000096
Episode 42327 total samples 2889188 train steps 89975
Length 120
Episode 42386 total samples 2891702 train steps 90053
Length 109
Episode 42449 total samples 2894227 train steps 90132
Length 89
Episode 42506 total samples 2896739 train steps 90211
Length 92
Episode 42564 total samples 2899256 train steps 90289
Length 107
Episode 42622 total samples 2901793 train steps 90369
Length 73
Episode 42681 total samples 2904345 train steps 90448
Length 105
Episode 42740 total samples 2906880 train steps 90527
Length 92
Episode 42800 total samples 2909434 train steps 90607
Length 136
Episode 42856 total samples 2911936 train steps 90685
Length 80
exploration 0.0999991000096
Episode 42916 total samples 2914447 train steps 90764
Length 78
Episode 42973 total samples 2916982 train steps 90843
Length 99
Episode 43028 total samples 2919491 train steps 90922
Length 86
Episode 43085 total samples 2921992 train steps 91000
Length 70
Episode 43143 total samples 2924509 train steps 91078
Length 86
Episode 43200 total samples 2927011 train steps 91157
Length 77
Episode 43259 total samples 2929541 train steps 91236
Length 72
Episode 43320 total samples 2932062 train steps 91314
Length 70
Episode 43379 total samples 2934605 train steps 91394
Length 100
Episode 43439 total samples 2937125 train steps 91473
Length 73
exploration 0.0999991000096
Episode 43496 total samples 2939649 train steps 91552
Length 69
Episode 43553 total samples 2942176 train steps 91630
Length 102
Episode 43612 total samples 2944712 train steps 91710
Length 73
Episode 43671 total samples 2947234 train steps 91789
Length 86
Episode 43729 total samples 2949763 train steps 91868
Length 103
Episode 43787 total samples 2952295 train steps 91947
Length 73
Episode 43844 total samples 2954835 train steps 92026
Length 93
Episode 43902 total samples 2957382 train steps 92106
Length 103
Episode 43959 total samples 2959914 train steps 92185
Length 71
Episode 44018 total samples 2962457 train steps 92264
Length 88
exploration 0.0999991000096
Episode 44075 total samples 2964992 train steps 92343
Length 90
Episode 44133 total samples 2967504 train steps 92422
Length 121
Episode 44193 total samples 2970029 train steps 92501
Length 104
Episode 44254 total samples 2972535 train steps 92579
Length 90
Episode 44310 total samples 2975051 train steps 92658
Length 81
Episode 44368 total samples 2977580 train steps 92737
Length 96
Episode 44426 total samples 2980158 train steps 92817
Length 158
Episode 44482 total samples 2982686 train steps 92896
Length 117
Episode 44542 total samples 2985209 train steps 92975
Length 92
Episode 44600 total samples 2987713 train steps 93054
Length 86
exploration 0.0999991000096
Episode 44657 total samples 2990250 train steps 93133
Length 92
Episode 44715 total samples 2992796 train steps 93212
Length 106
Episode 44774 total samples 2995312 train steps 93291
Length 84
Episode 44832 total samples 2997833 train steps 93370
Length 77
Episode 44890 total samples 3000364 train steps 93449
Length 135
Episode 44947 total samples 3002894 train steps 93528
Length 82
Episode 45006 total samples 3005418 train steps 93607
Length 87
Episode 45065 total samples 3007949 train steps 93686
Length 93
Episode 45123 total samples 3010482 train steps 93765
Length 81
exploration 0.0999991000096
Episode 45181 total samples 3013017 train steps 93844
Length 72
Episode 45240 total samples 3015539 train steps 93923
Length 62
Episode 45296 total samples 3018050 train steps 94002
Length 89
Episode 45354 total samples 3020553 train steps 94080
Length 71
Episode 45411 total samples 3023080 train steps 94159
Length 74
Episode 45468 total samples 3025601 train steps 94238
Length 83
Episode 45528 total samples 3028134 train steps 94317
Length 77
Episode 45586 total samples 3030656 train steps 94395
Length 90
Episode 45643 total samples 3033171 train steps 94474
Length 72
Episode 45700 total samples 3035697 train steps 94553
Length 88
exploration 0.0999991000096
Episode 45758 total samples 3038209 train steps 94632
Length 93
Episode 45815 total samples 3040717 train steps 94710
Length 87
Episode 45873 total samples 3043218 train steps 94788
Length 72
Episode 45931 total samples 3045757 train steps 94867
Length 98
Episode 45993 total samples 3048286 train steps 94946
Length 72
Episode 46053 total samples 3050823 train steps 95026
Length 82
Episode 46111 total samples 3053326 train steps 95104
Length 84
Episode 46167 total samples 3055846 train steps 95183
Length 120
Episode 46224 total samples 3058370 train steps 95262
Length 77
Episode 46280 total samples 3060901 train steps 95341
Length 87
exploration 0.0999991000096
Episode 46339 total samples 3063428 train steps 95420
Length 94
Episode 46397 total samples 3065951 train steps 95498
Length 123
Episode 46452 total samples 3068456 train steps 95577
Length 73
Episode 46510 total samples 3070978 train steps 95656
Length 94
Episode 46568 total samples 3073538 train steps 95736
Length 140
Episode 46627 total samples 3076061 train steps 95814
Length 96
Episode 46685 total samples 3078573 train steps 95893
Length 111
Episode 46740 total samples 3081118 train steps 95972
Length 105
Episode 46797 total samples 3083652 train steps 96052
Length 83
Episode 46855 total samples 3086186 train steps 96131
Length 70
exploration 0.0999991000096
Episode 46910 total samples 3088743 train steps 96211
Length 128
Episode 46969 total samples 3091268 train steps 96290
Length 76
Episode 47031 total samples 3093800 train steps 96369
Length 87
Episode 47092 total samples 3096307 train steps 96447
Length 68
Episode 47149 total samples 3098832 train steps 96526
Length 71
Episode 47205 total samples 3101335 train steps 96604
Length 96
Episode 47262 total samples 3103849 train steps 96683
Length 87
Episode 47323 total samples 3106388 train steps 96762
Length 100
Episode 47376 total samples 3108890 train steps 96840
Length 138
Episode 47435 total samples 3111432 train steps 96920
Length 89
exploration 0.0999991000096
Episode 47494 total samples 3113986 train steps 97000
Length 117
Episode 47555 total samples 3116527 train steps 97079
Length 84
Episode 47614 total samples 3119066 train steps 97158
Length 78
Episode 47670 total samples 3121597 train steps 97237
Length 65
Episode 47727 total samples 3124100 train steps 97316
Length 79
Episode 47785 total samples 3126644 train steps 97395
Length 97
Episode 47841 total samples 3129151 train steps 97473
Length 71
Episode 47902 total samples 3131654 train steps 97552
Length 95
Episode 47964 total samples 3134157 train steps 97630
Length 80
Episode 48025 total samples 3136678 train steps 97709
Length 105
exploration 0.0999991000096
Episode 48085 total samples 3139196 train steps 97787
Length 78
Episode 48146 total samples 3141729 train steps 97867
Length 98
Episode 48205 total samples 3144236 train steps 97945
Length 88
Episode 48263 total samples 3146766 train steps 98024
Length 85
Episode 48324 total samples 3149292 train steps 98103
Length 96
Episode 48384 total samples 3151813 train steps 98182
Length 106
Episode 48444 total samples 3154330 train steps 98260
Length 74
Episode 48504 total samples 3156840 train steps 98339
Length 107
Episode 48563 total samples 3159374 train steps 98418
Length 73
Episode 48622 total samples 3161891 train steps 98497
Length 71
exploration 0.0999991000096
Episode 48680 total samples 3164406 train steps 98575
Length 89
Episode 48738 total samples 3166923 train steps 98654
Length 70
Episode 48799 total samples 3169447 train steps 98733
Length 100
Episode 48855 total samples 3171951 train steps 98811
Length 109
Episode 48914 total samples 3174457 train steps 98889
Length 78
Episode 48974 total samples 3176978 train steps 98968
Length 87
Episode 49030 total samples 3179489 train steps 99047
Length 73
Episode 49086 total samples 3181999 train steps 99125
Length 70
Episode 49144 total samples 3184518 train steps 99204
Length 69
Episode 49204 total samples 3187032 train steps 99282
Length 90
exploration 0.0999991000096
Episode 49259 total samples 3189534 train steps 99360
Length 128
Episode 49317 total samples 3192051 train steps 99439
Length 74
Episode 49376 total samples 3194593 train steps 99519
Length 87
Episode 49435 total samples 3197131 train steps 99598
Length 103
Episode 49494 total samples 3199668 train steps 99677
Length 82
Episode 49551 total samples 3202180 train steps 99756
Length 72
Episode 49609 total samples 3204711 train steps 99835
Length 98
Episode 49666 total samples 3207241 train steps 99914
Length 68
Episode 49725 total samples 3209755 train steps 99992
Length 86
Episode 49782 total samples 3212281 train steps 100071
Length 117
exploration 0.0999991000096
Episode 49843 total samples 3214806 train steps 100150
Length 87
Episode 49901 total samples 3217317 train steps 100229
Length 111
Episode 49961 total samples 3219840 train steps 100307
Length 85
Episode 50021 total samples 3222365 train steps 100386
Length 111
Episode 50081 total samples 3224906 train steps 100466
Length 111
Episode 50138 total samples 3227448 train steps 100545
Length 85
Episode 50196 total samples 3229971 train steps 100624
Length 71
Episode 50254 total samples 3232501 train steps 100703
Length 78
Episode 50314 total samples 3235045 train steps 100783
Length 104
Episode 50372 total samples 3237560 train steps 100861
Length 104
exploration 0.0999991000096
Episode 50430 total samples 3240066 train steps 100940
Length 63
Episode 50487 total samples 3242606 train steps 101019
Length 88
Episode 50547 total samples 3245126 train steps 101098
Length 76
Episode 50602 total samples 3247639 train steps 101176
Length 120
Episode 50660 total samples 3250167 train steps 101255
Length 70
Episode 50715 total samples 3252674 train steps 101334
Length 71
Episode 50774 total samples 3255207 train steps 101413
Length 79
Episode 50830 total samples 3257720 train steps 101491
Length 79
Episode 50888 total samples 3260233 train steps 101570
Length 79
Episode 50947 total samples 3262786 train steps 101650
Length 114
exploration 0.0999991000096
Episode 51005 total samples 3265332 train steps 101729
Length 92
Episode 51066 total samples 3267841 train steps 101808
Length 93
Episode 51123 total samples 3270352 train steps 101886
Length 100
Episode 51181 total samples 3272884 train steps 101965
Length 64
Episode 51240 total samples 3275392 train steps 102043
Length 72
Episode 51300 total samples 3277912 train steps 102122
Length 81
Episode 51357 total samples 3280431 train steps 102201
Length 74
Episode 51417 total samples 3282937 train steps 102279
Length 83
Episode 51477 total samples 3285462 train steps 102358
Length 73
Episode 51538 total samples 3287987 train steps 102437
Length 89
exploration 0.0999991000096
Episode 51595 total samples 3290532 train steps 102517
Length 105
Episode 51655 total samples 3293041 train steps 102595
Length 62
Episode 51715 total samples 3295585 train steps 102675
Length 101
Episode 51775 total samples 3298108 train steps 102753
Length 87
Episode 51833 total samples 3300632 train steps 102832
Length 125
Episode 51893 total samples 3303159 train steps 102911
Length 82
Episode 51953 total samples 3305683 train steps 102990
Length 63
Episode 52011 total samples 3308219 train steps 103069
Length 83
Episode 52069 total samples 3310772 train steps 103149
Length 106
exploration 0.0999991000096
Episode 52129 total samples 3313289 train steps 103228
Length 86
Episode 52190 total samples 3315810 train steps 103307
Length 72
Episode 52248 total samples 3318341 train steps 103386
Length 70
Episode 52309 total samples 3320875 train steps 103465
Length 115
Episode 52364 total samples 3323396 train steps 103544
Length 88
Episode 52422 total samples 3325914 train steps 103622
Length 74
Episode 52480 total samples 3328452 train steps 103702
Length 105
Episode 52539 total samples 3330994 train steps 103781
Length 83
Episode 52598 total samples 3333495 train steps 103859
Length 72
Episode 52658 total samples 3336019 train steps 103938
Length 74
exploration 0.0999991000096
Episode 52715 total samples 3338521 train steps 104016
Length 125
Episode 52774 total samples 3341042 train steps 104095
Length 65
Episode 52835 total samples 3343580 train steps 104174
Length 115
Episode 52894 total samples 3346120 train steps 104254
Length 95
Episode 52954 total samples 3348652 train steps 104333
Length 98
Episode 53013 total samples 3351174 train steps 104412
Length 107
Episode 53074 total samples 3353709 train steps 104491
Length 76
Episode 53135 total samples 3356210 train steps 104569
Length 87
Episode 53195 total samples 3358739 train steps 104648
Length 89
Episode 53254 total samples 3361251 train steps 104727
Length 77
exploration 0.0999991000096
Episode 53313 total samples 3363788 train steps 104806
Length 93
Episode 53374 total samples 3366323 train steps 104885
Length 82
Episode 53432 total samples 3368852 train steps 104964
Length 86
Episode 53491 total samples 3371365 train steps 105043
Length 93
Episode 53552 total samples 3373920 train steps 105122
Length 122
Episode 53609 total samples 3376426 train steps 105201
Length 84
Episode 53669 total samples 3378927 train steps 105279
Length 71
Episode 53728 total samples 3381471 train steps 105358
Length 101
Episode 53788 total samples 3383996 train steps 105437
Length 71
Episode 53846 total samples 3386499 train steps 105516
Length 97
exploration 0.0999991000096
Episode 53904 total samples 3389007 train steps 105594
Length 97
Episode 53963 total samples 3391532 train steps 105673
Length 102
Episode 54023 total samples 3394080 train steps 105752
Length 97
Episode 54082 total samples 3396627 train steps 105832
Length 98
---------------------------------------------------------------------------
InternalError Traceback (most recent call last)
<ipython-input-4-abe2a70abffd> in <module>()
----> 1 car.deepq_learning(num_iter = 100000, max_steps = 500000)#, reset_replay_memory=True)
/home/ga24pug/DeepRL/q_learning_sarsa/dqn_learning.py in deepq_learning(self, num_iter, max_steps, max_learning_steps, learning_rate, reset_replay_memory)
267 action = self.policy(state_tensor,
268 mode=self.policy_mode,
--> 269 deepQ=True)
270 # action = self.policy(state, mode=self.policy_mode, deepQ=True)
271 # episode.append((state, action, reward))
/home/ga24pug/DeepRL/q_learning_sarsa/dqn_learning.py in policy(self, state, mode, deepQ)
177 if mode == 'deterministic' and not explore:
178 if deepQ:
--> 179 q = self.qnn.evaluate_all_actions(state)
180 # print(state, q)
181 return np.argmax(q.squeeze())#np.random.choice(np.argwhere(q == np.amax(q)).flatten())
/home/ga24pug/DeepRL/q_learning_sarsa/qnn.pyc in evaluate_all_actions(self, states)
666 # print(states, states.shape)
667 feed_dict = {self.s: states, self.keep_prob: 1.0, self.phase_train: False}
--> 668 return self.sess.run(self.q_all, feed_dict=feed_dict)
669
670 # this does not seem to work
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in run(self, fetches, feed_dict, options, run_metadata)
370 try:
371 result = self._run(None, fetches, feed_dict, options_ptr,
--> 372 run_metadata_ptr)
373 if run_metadata:
374 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _run(self, handle, fetches, feed_dict, options, run_metadata)
634 try:
635 results = self._do_run(handle, target_list, unique_fetches,
--> 636 feed_dict_string, options, run_metadata)
637 finally:
638 # The movers are no longer used. Delete them.
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
706 if handle is None:
707 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
--> 708 target_list, options, run_metadata)
709 else:
710 return self._do_call(_prun_fn, self._session, handle, feed_dict,
/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/client/session.pyc in _do_call(self, fn, *args)
726 except KeyError:
727 pass
--> 728 raise type(e)(node_def, op, message)
729
730 def _extend_graph(self):
InternalError: Blas SGEMM launch failed : a.shape=(1, 7744), b.shape=(7744, 512), m=1, n=512, k=7744
[[Node: fc_layer1/Wx_plus_b/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](Reshape, fc_layer1/weights/truediv)]]
Caused by op u'fc_layer1/Wx_plus_b/MatMul', defined at:
File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
"__main__", fname, loader, pkg_name)
File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
exec code in run_globals
File "/home/ga24pug/mypython-deeprl/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
app.launch_new_instance()
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/traitlets/config/application.py", line 596, in launch_instance
app.start()
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 442, in start
ioloop.IOLoop.instance().start()
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py", line 160, in start
super(ZMQIOLoop, self).start()
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tornado/ioloop.py", line 883, in start
handler_func(fd_obj, events)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 433, in _handle_events
self._handle_recv()
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 465, in _handle_recv
self._run_callback(callback, msg)
File "/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py", line 407, in _run_callback
callback(*args, **kwargs)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
return fn(*args, **kwargs)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 276, in dispatcher
return self.dispatch_shell(stream, msg)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 228, in dispatch_shell
handler(stream, idents, msg)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 391, in execute_request
user_expressions, allow_stdin)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 199, in do_execute
shell.run_cell(code, store_history=store_history, silent=silent)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2723, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2825, in run_ast_nodes
if self.run_code(code, result):
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-3-a572adf0a59c>", line 23, in <module>
pretrain_steps = 0
File "dqn_learning.py", line 161, in __init__
pretrain_steps=pretrain_steps
File "qnn.py", line 433, in __init__
last_hidden = nn_dual_layer_with_decay(conv3_flat, last_conv_layer_output_size, 512, 'fc_layer'+str(1), ema_ops_list, decay=ema_decay_rate)
File "qnn.py", line 266, in nn_dual_layer_with_decay
preactivate_s = tf.matmul(input_tensor_list[0], weights) + biases
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/ops/math_ops.py", line 1209, in matmul
name=name)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/ops/gen_math_ops.py", line 1178, in _mat_mul
transpose_b=transpose_b, name=name)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/ops/op_def_library.py", line 704, in apply_op
op_def=op_def)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2260, in create_op
original_op=self._default_original_op, op_def=op_def)
File "/home/ga24pug/mypython-deeprl/local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1230, in __init__
self._traceback = _extract_stack()
In [ ]:
In [ ]: