Gradient Descent Implementation

This is the code from Bogotobogo to practice the usage


In [17]:
import numpy as np
import random
import sklearn
import pylab
from scipy import stats
from sklearn.datasets.samples_generator import make_regression

In [18]:
def gradient_descent1(alpha, x, y, ep=0.0001, max_iter=10000):
    converged = False
    iteration = 0
    m = x.shape[0] # Number of samples
    
    # Initialize the theta
    t0 = np.random.random(x.shape[1])
    t1 = np.random.random(x.shape[1])
    
    # Total error, J(theta)
    J = sum([(t0 + t1*x[i] - y[i]) ** 2 for i in range(m)])
    
    # Iterate Loop
    while not converged:
        # For each training sample, compute the gradient (d/d_theta j(theta))
        grad0 = 1.0/m * sum([(t0 + t1*x[i] - y[i] for i in range(m))])
        grad1 = 1.0/m * sum([(t0 + t1*x[i] - y[i]) * x[i] for i in range(m)])
        
        # Update the theta_temp
        temp0 = t0 - alpha * grad0
        temp1 = t1 - alpha * grad1
        
        # Update theta
        t0 = temp0
        t1 = temp1
        
        # Mean squared error
        e = sum( [ (t0 + t1*x[i] - y[i])**2  for i in range(m)] )
        
        if abs(J - e) <= ep:
            print("Converged, iterations: " + str(iterations) + "!!!")
            converged = True
        
        J = e # Update error
        iteration += 1
        
        if iteration == max_iter:
            print('Max interactions exceeded!')
            converged = True
            
    return t0, t1

In [19]:
def main1():
    x, y = make_regression(n_samples=100, n_features=1, n_informative=1,
                          random_state=0, noise=35)
    print('x.shape = %s y.shape = %s' %(x.shape, y.shape))
    
    alpha = 0.01 # Learning rate
    ep = 0.01 # Convergence criteria
    
    # Call gradient decent, and get intercept(=theta0)  and slope(=theta1)
    theta0, theta1 = gradient_descent1(alpha, x, y, ep, max_iter=1000)
    print(('theta0 = %s theta1 = %s') %(theta0, theta1))
    
    # Check with scipy linear regression
    slope, intercept, r_value, p_value, slope_std_error = stats.linregress(x[:,0], y)
    print(('intercept = %s slope = %s') %(intercept, slope))
    
    # Plot
    for i in range(x.shape[0]):
        y_predict = theta0 + theta1 * x
        
    pylab.plot(x,y,'o')
    pylab.plot(x,y_predict,'k-')
    pylab.show()
    print("Done!")

In [20]:
def gradient_descent2(alpha, x, y, numIterations):
    m = x.shape[0] # Number of samples
    theta = np.ones(2)
    x_transpose = x.transpose()
    for i in range(0, numIterations):
        hypothesis = np.dot(x, theta)
        loss = hypothesis - y
        J = np.sum(loss ** 2) / (2 * m) # Cost 
        print("iter %s | J: %.3f" % (iter, J) )
        
        gradient = np.dot(x_transpose, loss) / m
        theta = theta - alpha * gradient # Update 
    
    return theta

In [22]:
def main2():
    x, y = make_regression(n_samples=100, n_features=1,
                          n_informative=1, random_state=0,
                          noise=35)
    m, n = np.shape(x)
    x = np.c_[ np.ones(m), x] # Insert column
    alpha = 0.01 # learning rate
    theta = gradient_descent2(alpha, x, y, 1000)
    
    # plot
    for i in range(x.shape[1]):
        y_predict = theta[0] + theta[1]*x 
    pylab.plot(x[:,1],y,'o')
    pylab.plot(x,y_predict,'k-')
    pylab.show()
    print "Done!"

In [24]:
main2()


iter <built-in function iter> | J: 1604.873
iter <built-in function iter> | J: 1586.636
iter <built-in function iter> | J: 1568.768
iter <built-in function iter> | J: 1551.261
iter <built-in function iter> | J: 1534.107
iter <built-in function iter> | J: 1517.301
iter <built-in function iter> | J: 1500.833
iter <built-in function iter> | J: 1484.699
iter <built-in function iter> | J: 1468.890
iter <built-in function iter> | J: 1453.401
iter <built-in function iter> | J: 1438.224
iter <built-in function iter> | J: 1423.354
iter <built-in function iter> | J: 1408.785
iter <built-in function iter> | J: 1394.509
iter <built-in function iter> | J: 1380.522
iter <built-in function iter> | J: 1366.817
iter <built-in function iter> | J: 1353.389
iter <built-in function iter> | J: 1340.232
iter <built-in function iter> | J: 1327.341
iter <built-in function iter> | J: 1314.710
iter <built-in function iter> | J: 1302.334
iter <built-in function iter> | J: 1290.207
iter <built-in function iter> | J: 1278.325
iter <built-in function iter> | J: 1266.683
iter <built-in function iter> | J: 1255.276
iter <built-in function iter> | J: 1244.100
iter <built-in function iter> | J: 1233.148
iter <built-in function iter> | J: 1222.418
iter <built-in function iter> | J: 1211.904
iter <built-in function iter> | J: 1201.602
iter <built-in function iter> | J: 1191.507
iter <built-in function iter> | J: 1181.617
iter <built-in function iter> | J: 1171.926
iter <built-in function iter> | J: 1162.430
iter <built-in function iter> | J: 1153.126
iter <built-in function iter> | J: 1144.009
iter <built-in function iter> | J: 1135.076
iter <built-in function iter> | J: 1126.323
iter <built-in function iter> | J: 1117.747
iter <built-in function iter> | J: 1109.343
iter <built-in function iter> | J: 1101.109
iter <built-in function iter> | J: 1093.041
iter <built-in function iter> | J: 1085.135
iter <built-in function iter> | J: 1077.389
iter <built-in function iter> | J: 1069.799
iter <built-in function iter> | J: 1062.361
iter <built-in function iter> | J: 1055.074
iter <built-in function iter> | J: 1047.933
iter <built-in function iter> | J: 1040.936
iter <built-in function iter> | J: 1034.080
iter <built-in function iter> | J: 1027.363
iter <built-in function iter> | J: 1020.780
iter <built-in function iter> | J: 1014.330
iter <built-in function iter> | J: 1008.010
iter <built-in function iter> | J: 1001.817
iter <built-in function iter> | J: 995.749
iter <built-in function iter> | J: 989.803
iter <built-in function iter> | J: 983.976
iter <built-in function iter> | J: 978.267
iter <built-in function iter> | J: 972.673
iter <built-in function iter> | J: 967.191
iter <built-in function iter> | J: 961.820
iter <built-in function iter> | J: 956.557
iter <built-in function iter> | J: 951.399
iter <built-in function iter> | J: 946.346
iter <built-in function iter> | J: 941.394
iter <built-in function iter> | J: 936.542
iter <built-in function iter> | J: 931.787
iter <built-in function iter> | J: 927.128
iter <built-in function iter> | J: 922.563
iter <built-in function iter> | J: 918.089
iter <built-in function iter> | J: 913.705
iter <built-in function iter> | J: 909.410
iter <built-in function iter> | J: 905.201
iter <built-in function iter> | J: 901.076
iter <built-in function iter> | J: 897.035
iter <built-in function iter> | J: 893.075
iter <built-in function iter> | J: 889.194
iter <built-in function iter> | J: 885.391
iter <built-in function iter> | J: 881.665
iter <built-in function iter> | J: 878.013
iter <built-in function iter> | J: 874.435
iter <built-in function iter> | J: 870.929
iter <built-in function iter> | J: 867.494
iter <built-in function iter> | J: 864.127
iter <built-in function iter> | J: 860.828
iter <built-in function iter> | J: 857.595
iter <built-in function iter> | J: 854.427
iter <built-in function iter> | J: 851.323
iter <built-in function iter> | J: 848.281
iter <built-in function iter> | J: 845.300
iter <built-in function iter> | J: 842.379
iter <built-in function iter> | J: 839.517
iter <built-in function iter> | J: 836.712
iter <built-in function iter> | J: 833.964
iter <built-in function iter> | J: 831.270
iter <built-in function iter> | J: 828.631
iter <built-in function iter> | J: 826.045
iter <built-in function iter> | J: 823.510
iter <built-in function iter> | J: 821.027
iter <built-in function iter> | J: 818.593
iter <built-in function iter> | J: 816.208
iter <built-in function iter> | J: 813.871
iter <built-in function iter> | J: 811.581
iter <built-in function iter> | J: 809.337
iter <built-in function iter> | J: 807.138
iter <built-in function iter> | J: 804.983
iter <built-in function iter> | J: 802.871
iter <built-in function iter> | J: 800.801
iter <built-in function iter> | J: 798.773
iter <built-in function iter> | J: 796.786
iter <built-in function iter> | J: 794.838
iter <built-in function iter> | J: 792.930
iter <built-in function iter> | J: 791.060
iter <built-in function iter> | J: 789.227
iter <built-in function iter> | J: 787.431
iter <built-in function iter> | J: 785.671
iter <built-in function iter> | J: 783.946
iter <built-in function iter> | J: 782.256
iter <built-in function iter> | J: 780.600
iter <built-in function iter> | J: 778.977
iter <built-in function iter> | J: 777.386
iter <built-in function iter> | J: 775.828
iter <built-in function iter> | J: 774.300
iter <built-in function iter> | J: 772.803
iter <built-in function iter> | J: 771.336
iter <built-in function iter> | J: 769.899
iter <built-in function iter> | J: 768.490
iter <built-in function iter> | J: 767.110
iter <built-in function iter> | J: 765.757
iter <built-in function iter> | J: 764.431
iter <built-in function iter> | J: 763.132
iter <built-in function iter> | J: 761.859
iter <built-in function iter> | J: 760.611
iter <built-in function iter> | J: 759.388
iter <built-in function iter> | J: 758.190
iter <built-in function iter> | J: 757.015
iter <built-in function iter> | J: 755.865
iter <built-in function iter> | J: 754.737
iter <built-in function iter> | J: 753.632
iter <built-in function iter> | J: 752.548
iter <built-in function iter> | J: 751.487
iter <built-in function iter> | J: 750.447
iter <built-in function iter> | J: 749.427
iter <built-in function iter> | J: 748.428
iter <built-in function iter> | J: 747.449
iter <built-in function iter> | J: 746.490
iter <built-in function iter> | J: 745.550
iter <built-in function iter> | J: 744.628
iter <built-in function iter> | J: 743.725
iter <built-in function iter> | J: 742.840
iter <built-in function iter> | J: 741.973
iter <built-in function iter> | J: 741.123
iter <built-in function iter> | J: 740.290
iter <built-in function iter> | J: 739.474
iter <built-in function iter> | J: 738.674
iter <built-in function iter> | J: 737.890
iter <built-in function iter> | J: 737.121
iter <built-in function iter> | J: 736.368
iter <built-in function iter> | J: 735.630
iter <built-in function iter> | J: 734.907
iter <built-in function iter> | J: 734.198
iter <built-in function iter> | J: 733.504
iter <built-in function iter> | J: 732.823
iter <built-in function iter> | J: 732.156
iter <built-in function iter> | J: 731.502
iter <built-in function iter> | J: 730.861
iter <built-in function iter> | J: 730.234
iter <built-in function iter> | J: 729.618
iter <built-in function iter> | J: 729.015
iter <built-in function iter> | J: 728.424
iter <built-in function iter> | J: 727.845
iter <built-in function iter> | J: 727.277
iter <built-in function iter> | J: 726.720
iter <built-in function iter> | J: 726.175
iter <built-in function iter> | J: 725.641
iter <built-in function iter> | J: 725.117
iter <built-in function iter> | J: 724.604
iter <built-in function iter> | J: 724.101
iter <built-in function iter> | J: 723.608
iter <built-in function iter> | J: 723.124
iter <built-in function iter> | J: 722.651
iter <built-in function iter> | J: 722.187
iter <built-in function iter> | J: 721.732
iter <built-in function iter> | J: 721.286
iter <built-in function iter> | J: 720.849
iter <built-in function iter> | J: 720.421
iter <built-in function iter> | J: 720.002
iter <built-in function iter> | J: 719.590
iter <built-in function iter> | J: 719.187
iter <built-in function iter> | J: 718.792
iter <built-in function iter> | J: 718.405
iter <built-in function iter> | J: 718.026
iter <built-in function iter> | J: 717.654
iter <built-in function iter> | J: 717.289
iter <built-in function iter> | J: 716.932
iter <built-in function iter> | J: 716.582
iter <built-in function iter> | J: 716.239
iter <built-in function iter> | J: 715.903
iter <built-in function iter> | J: 715.573
iter <built-in function iter> | J: 715.250
iter <built-in function iter> | J: 714.933
iter <built-in function iter> | J: 714.623
iter <built-in function iter> | J: 714.319
iter <built-in function iter> | J: 714.021
iter <built-in function iter> | J: 713.729
iter <built-in function iter> | J: 713.443
iter <built-in function iter> | J: 713.162
iter <built-in function iter> | J: 712.887
iter <built-in function iter> | J: 712.618
iter <built-in function iter> | J: 712.353
iter <built-in function iter> | J: 712.095
iter <built-in function iter> | J: 711.841
iter <built-in function iter> | J: 711.592
iter <built-in function iter> | J: 711.348
iter <built-in function iter> | J: 711.109
iter <built-in function iter> | J: 710.875
iter <built-in function iter> | J: 710.646
iter <built-in function iter> | J: 710.421
iter <built-in function iter> | J: 710.200
iter <built-in function iter> | J: 709.984
iter <built-in function iter> | J: 709.773
iter <built-in function iter> | J: 709.565
iter <built-in function iter> | J: 709.362
iter <built-in function iter> | J: 709.162
iter <built-in function iter> | J: 708.967
iter <built-in function iter> | J: 708.775
iter <built-in function iter> | J: 708.588
iter <built-in function iter> | J: 708.404
iter <built-in function iter> | J: 708.223
iter <built-in function iter> | J: 708.047
iter <built-in function iter> | J: 707.873
iter <built-in function iter> | J: 707.704
iter <built-in function iter> | J: 707.537
iter <built-in function iter> | J: 707.374
iter <built-in function iter> | J: 707.214
iter <built-in function iter> | J: 707.058
iter <built-in function iter> | J: 706.904
iter <built-in function iter> | J: 706.753
iter <built-in function iter> | J: 706.606
iter <built-in function iter> | J: 706.461
iter <built-in function iter> | J: 706.320
iter <built-in function iter> | J: 706.181
iter <built-in function iter> | J: 706.044
iter <built-in function iter> | J: 705.911
iter <built-in function iter> | J: 705.780
iter <built-in function iter> | J: 705.652
iter <built-in function iter> | J: 705.526
iter <built-in function iter> | J: 705.403
iter <built-in function iter> | J: 705.282
iter <built-in function iter> | J: 705.164
iter <built-in function iter> | J: 705.048
iter <built-in function iter> | J: 704.934
iter <built-in function iter> | J: 704.823
iter <built-in function iter> | J: 704.714
iter <built-in function iter> | J: 704.607
iter <built-in function iter> | J: 704.502
iter <built-in function iter> | J: 704.399
iter <built-in function iter> | J: 704.298
iter <built-in function iter> | J: 704.199
iter <built-in function iter> | J: 704.103
iter <built-in function iter> | J: 704.008
iter <built-in function iter> | J: 703.915
iter <built-in function iter> | J: 703.823
iter <built-in function iter> | J: 703.734
iter <built-in function iter> | J: 703.646
iter <built-in function iter> | J: 703.561
iter <built-in function iter> | J: 703.476
iter <built-in function iter> | J: 703.394
iter <built-in function iter> | J: 703.313
iter <built-in function iter> | J: 703.234
iter <built-in function iter> | J: 703.156
iter <built-in function iter> | J: 703.080
iter <built-in function iter> | J: 703.005
iter <built-in function iter> | J: 702.932
iter <built-in function iter> | J: 702.860
iter <built-in function iter> | J: 702.790
iter <built-in function iter> | J: 702.721
iter <built-in function iter> | J: 702.654
iter <built-in function iter> | J: 702.588
iter <built-in function iter> | J: 702.523
iter <built-in function iter> | J: 702.459
iter <built-in function iter> | J: 702.397
iter <built-in function iter> | J: 702.336
iter <built-in function iter> | J: 702.276
iter <built-in function iter> | J: 702.217
iter <built-in function iter> | J: 702.159
iter <built-in function iter> | J: 702.103
iter <built-in function iter> | J: 702.048
iter <built-in function iter> | J: 701.994
iter <built-in function iter> | J: 701.940
iter <built-in function iter> | J: 701.888
iter <built-in function iter> | J: 701.837
iter <built-in function iter> | J: 701.787
iter <built-in function iter> | J: 701.738
iter <built-in function iter> | J: 701.690
iter <built-in function iter> | J: 701.643
iter <built-in function iter> | J: 701.597
iter <built-in function iter> | J: 701.552
iter <built-in function iter> | J: 701.507
iter <built-in function iter> | J: 701.464
iter <built-in function iter> | J: 701.421
iter <built-in function iter> | J: 701.379
iter <built-in function iter> | J: 701.338
iter <built-in function iter> | J: 701.298
iter <built-in function iter> | J: 701.259
iter <built-in function iter> | J: 701.220
iter <built-in function iter> | J: 701.182
iter <built-in function iter> | J: 701.145
iter <built-in function iter> | J: 701.109
iter <built-in function iter> | J: 701.073
iter <built-in function iter> | J: 701.038
iter <built-in function iter> | J: 701.004
iter <built-in function iter> | J: 700.971
iter <built-in function iter> | J: 700.938
iter <built-in function iter> | J: 700.905
iter <built-in function iter> | J: 700.874
iter <built-in function iter> | J: 700.843
iter <built-in function iter> | J: 700.812
iter <built-in function iter> | J: 700.783
iter <built-in function iter> | J: 700.753
iter <built-in function iter> | J: 700.725
iter <built-in function iter> | J: 700.697
iter <built-in function iter> | J: 700.669
iter <built-in function iter> | J: 700.642
iter <built-in function iter> | J: 700.616
iter <built-in function iter> | J: 700.590
iter <built-in function iter> | J: 700.565
iter <built-in function iter> | J: 700.540
iter <built-in function iter> | J: 700.515
iter <built-in function iter> | J: 700.492
iter <built-in function iter> | J: 700.468
iter <built-in function iter> | J: 700.445
iter <built-in function iter> | J: 700.423
iter <built-in function iter> | J: 700.401
iter <built-in function iter> | J: 700.379
iter <built-in function iter> | J: 700.358
iter <built-in function iter> | J: 700.337
iter <built-in function iter> | J: 700.317
iter <built-in function iter> | J: 700.297
iter <built-in function iter> | J: 700.277
iter <built-in function iter> | J: 700.258
iter <built-in function iter> | J: 700.239
iter <built-in function iter> | J: 700.221
iter <built-in function iter> | J: 700.202
iter <built-in function iter> | J: 700.185
iter <built-in function iter> | J: 700.167
iter <built-in function iter> | J: 700.150
iter <built-in function iter> | J: 700.134
iter <built-in function iter> | J: 700.117
iter <built-in function iter> | J: 700.101
iter <built-in function iter> | J: 700.085
iter <built-in function iter> | J: 700.070
iter <built-in function iter> | J: 700.055
iter <built-in function iter> | J: 700.040
iter <built-in function iter> | J: 700.026
iter <built-in function iter> | J: 700.011
iter <built-in function iter> | J: 699.997
iter <built-in function iter> | J: 699.984
iter <built-in function iter> | J: 699.970
iter <built-in function iter> | J: 699.957
iter <built-in function iter> | J: 699.944
iter <built-in function iter> | J: 699.932
iter <built-in function iter> | J: 699.919
iter <built-in function iter> | J: 699.907
iter <built-in function iter> | J: 699.895
iter <built-in function iter> | J: 699.883
iter <built-in function iter> | J: 699.872
iter <built-in function iter> | J: 699.861
iter <built-in function iter> | J: 699.850
iter <built-in function iter> | J: 699.839
iter <built-in function iter> | J: 699.828
iter <built-in function iter> | J: 699.818
iter <built-in function iter> | J: 699.808
iter <built-in function iter> | J: 699.798
iter <built-in function iter> | J: 699.788
iter <built-in function iter> | J: 699.779
iter <built-in function iter> | J: 699.769
iter <built-in function iter> | J: 699.760
iter <built-in function iter> | J: 699.751
iter <built-in function iter> | J: 699.742
iter <built-in function iter> | J: 699.734
iter <built-in function iter> | J: 699.725
iter <built-in function iter> | J: 699.717
iter <built-in function iter> | J: 699.709
iter <built-in function iter> | J: 699.701
iter <built-in function iter> | J: 699.693
iter <built-in function iter> | J: 699.685
iter <built-in function iter> | J: 699.677
iter <built-in function iter> | J: 699.670
iter <built-in function iter> | J: 699.663
iter <built-in function iter> | J: 699.656
iter <built-in function iter> | J: 699.649
iter <built-in function iter> | J: 699.642
iter <built-in function iter> | J: 699.635
iter <built-in function iter> | J: 699.629
iter <built-in function iter> | J: 699.622
iter <built-in function iter> | J: 699.616
iter <built-in function iter> | J: 699.610
iter <built-in function iter> | J: 699.604
iter <built-in function iter> | J: 699.598
iter <built-in function iter> | J: 699.592
iter <built-in function iter> | J: 699.586
iter <built-in function iter> | J: 699.581
iter <built-in function iter> | J: 699.575
iter <built-in function iter> | J: 699.570
iter <built-in function iter> | J: 699.564
iter <built-in function iter> | J: 699.559
iter <built-in function iter> | J: 699.554
iter <built-in function iter> | J: 699.549
iter <built-in function iter> | J: 699.544
iter <built-in function iter> | J: 699.540
iter <built-in function iter> | J: 699.535
iter <built-in function iter> | J: 699.530
iter <built-in function iter> | J: 699.526
iter <built-in function iter> | J: 699.521
iter <built-in function iter> | J: 699.517
iter <built-in function iter> | J: 699.513
iter <built-in function iter> | J: 699.509
iter <built-in function iter> | J: 699.505
iter <built-in function iter> | J: 699.501
iter <built-in function iter> | J: 699.497
iter <built-in function iter> | J: 699.493
iter <built-in function iter> | J: 699.489
iter <built-in function iter> | J: 699.485
iter <built-in function iter> | J: 699.482
iter <built-in function iter> | J: 699.478
iter <built-in function iter> | J: 699.475
iter <built-in function iter> | J: 699.471
iter <built-in function iter> | J: 699.468
iter <built-in function iter> | J: 699.465
iter <built-in function iter> | J: 699.462
iter <built-in function iter> | J: 699.458
iter <built-in function iter> | J: 699.455
iter <built-in function iter> | J: 699.452
iter <built-in function iter> | J: 699.449
iter <built-in function iter> | J: 699.446
iter <built-in function iter> | J: 699.444
iter <built-in function iter> | J: 699.441
iter <built-in function iter> | J: 699.438
iter <built-in function iter> | J: 699.435
iter <built-in function iter> | J: 699.433
iter <built-in function iter> | J: 699.430
iter <built-in function iter> | J: 699.428
iter <built-in function iter> | J: 699.425
iter <built-in function iter> | J: 699.423
iter <built-in function iter> | J: 699.420
iter <built-in function iter> | J: 699.418
iter <built-in function iter> | J: 699.416
iter <built-in function iter> | J: 699.413
iter <built-in function iter> | J: 699.411
iter <built-in function iter> | J: 699.409
iter <built-in function iter> | J: 699.407
iter <built-in function iter> | J: 699.405
iter <built-in function iter> | J: 699.403
iter <built-in function iter> | J: 699.401
iter <built-in function iter> | J: 699.399
iter <built-in function iter> | J: 699.397
iter <built-in function iter> | J: 699.395
iter <built-in function iter> | J: 699.393
iter <built-in function iter> | J: 699.391
iter <built-in function iter> | J: 699.390
iter <built-in function iter> | J: 699.388
iter <built-in function iter> | J: 699.386
iter <built-in function iter> | J: 699.384
iter <built-in function iter> | J: 699.383
iter <built-in function iter> | J: 699.381
iter <built-in function iter> | J: 699.380
iter <built-in function iter> | J: 699.378
iter <built-in function iter> | J: 699.377
iter <built-in function iter> | J: 699.375
iter <built-in function iter> | J: 699.374
iter <built-in function iter> | J: 699.372
iter <built-in function iter> | J: 699.371
iter <built-in function iter> | J: 699.369
iter <built-in function iter> | J: 699.368
iter <built-in function iter> | J: 699.367
iter <built-in function iter> | J: 699.365
iter <built-in function iter> | J: 699.364
iter <built-in function iter> | J: 699.363
iter <built-in function iter> | J: 699.362
iter <built-in function iter> | J: 699.361
iter <built-in function iter> | J: 699.359
iter <built-in function iter> | J: 699.358
iter <built-in function iter> | J: 699.357
iter <built-in function iter> | J: 699.356
iter <built-in function iter> | J: 699.355
iter <built-in function iter> | J: 699.354
iter <built-in function iter> | J: 699.353
iter <built-in function iter> | J: 699.352
iter <built-in function iter> | J: 699.351
iter <built-in function iter> | J: 699.350
iter <built-in function iter> | J: 699.349
iter <built-in function iter> | J: 699.348
iter <built-in function iter> | J: 699.347
iter <built-in function iter> | J: 699.346
iter <built-in function iter> | J: 699.345
iter <built-in function iter> | J: 699.344
iter <built-in function iter> | J: 699.343
iter <built-in function iter> | J: 699.343
iter <built-in function iter> | J: 699.342
iter <built-in function iter> | J: 699.341
iter <built-in function iter> | J: 699.340
iter <built-in function iter> | J: 699.339
iter <built-in function iter> | J: 699.339
iter <built-in function iter> | J: 699.338
iter <built-in function iter> | J: 699.337
iter <built-in function iter> | J: 699.336
iter <built-in function iter> | J: 699.336
iter <built-in function iter> | J: 699.335
iter <built-in function iter> | J: 699.334
iter <built-in function iter> | J: 699.334
iter <built-in function iter> | J: 699.333
iter <built-in function iter> | J: 699.332
iter <built-in function iter> | J: 699.332
iter <built-in function iter> | J: 699.331
iter <built-in function iter> | J: 699.331
iter <built-in function iter> | J: 699.330
iter <built-in function iter> | J: 699.329
iter <built-in function iter> | J: 699.329
iter <built-in function iter> | J: 699.328
iter <built-in function iter> | J: 699.328
iter <built-in function iter> | J: 699.327
iter <built-in function iter> | J: 699.327
iter <built-in function iter> | J: 699.326
iter <built-in function iter> | J: 699.326
iter <built-in function iter> | J: 699.325
iter <built-in function iter> | J: 699.325
iter <built-in function iter> | J: 699.324
iter <built-in function iter> | J: 699.324
iter <built-in function iter> | J: 699.323
iter <built-in function iter> | J: 699.323
iter <built-in function iter> | J: 699.322
iter <built-in function iter> | J: 699.322
iter <built-in function iter> | J: 699.322
iter <built-in function iter> | J: 699.321
iter <built-in function iter> | J: 699.321
iter <built-in function iter> | J: 699.320
iter <built-in function iter> | J: 699.320
iter <built-in function iter> | J: 699.320
iter <built-in function iter> | J: 699.319
iter <built-in function iter> | J: 699.319
iter <built-in function iter> | J: 699.318
iter <built-in function iter> | J: 699.318
iter <built-in function iter> | J: 699.318
iter <built-in function iter> | J: 699.317
iter <built-in function iter> | J: 699.317
iter <built-in function iter> | J: 699.317
iter <built-in function iter> | J: 699.316
iter <built-in function iter> | J: 699.316
iter <built-in function iter> | J: 699.316
iter <built-in function iter> | J: 699.316
iter <built-in function iter> | J: 699.315
iter <built-in function iter> | J: 699.315
iter <built-in function iter> | J: 699.315
iter <built-in function iter> | J: 699.314
iter <built-in function iter> | J: 699.314
iter <built-in function iter> | J: 699.314
iter <built-in function iter> | J: 699.314
iter <built-in function iter> | J: 699.313
iter <built-in function iter> | J: 699.313
iter <built-in function iter> | J: 699.313
iter <built-in function iter> | J: 699.313
iter <built-in function iter> | J: 699.312
iter <built-in function iter> | J: 699.312
iter <built-in function iter> | J: 699.312
iter <built-in function iter> | J: 699.312
iter <built-in function iter> | J: 699.311
iter <built-in function iter> | J: 699.311
iter <built-in function iter> | J: 699.311
iter <built-in function iter> | J: 699.311
iter <built-in function iter> | J: 699.311
iter <built-in function iter> | J: 699.310
iter <built-in function iter> | J: 699.310
iter <built-in function iter> | J: 699.310
iter <built-in function iter> | J: 699.310
iter <built-in function iter> | J: 699.310
iter <built-in function iter> | J: 699.309
iter <built-in function iter> | J: 699.309
iter <built-in function iter> | J: 699.309
iter <built-in function iter> | J: 699.309
iter <built-in function iter> | J: 699.309
iter <built-in function iter> | J: 699.309
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.308
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.307
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.306
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.305
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.304
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.303
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.302
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.301
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
iter <built-in function iter> | J: 699.300
Done!

In [ ]: