

In [7]:

    
for t in range(500):
    # forward pass: compute predicted y
    h = np.dot(x, w1)
    h_relu = np.maximum(h, 0)
    y_pred = np.dot(h_relu, w2)
    
    # compute and print loss(MSE)
    loss = np.square(y_pred - y).sum()
    print(t, loss)
    
    # backprop to compute gradients of w1 and w2 with respect to loss
    grad_y_pred = 2.0 * (y_pred - y)
    grad_w2 = h_relu.T.dot(grad_y_pred)
    grad_h_relu = grad_y_pred.dot(w2.T)
    grad_h = grad_h_relu.copy()
    grad_h[h < 0] = 0
    grad_w1 = x.T.dot(grad_h)
    
    # update weights
    w1 -= learning_rate * grad_w1
    w2 -= learning_rate * grad_w2









    



0 27407862.0447
1 22082063.3269
2 20228925.8908
3 19028478.405
4 17018382.6404
5 13941151.6238
6 10325033.846
7 7066755.32658
8 4599930.18936
9 2968711.83062
10 1957345.98637
11 1350167.03674
12 981445.150781
13 750761.955043
14 599144.630053
15 493816.122237
16 416543.974428
17 357178.340398
18 309764.277999
19 270895.591036
20 238459.693783
21 210939.909213
22 187338.849467
23 166916.476332
24 149144.350621
25 133600.378667
26 119958.999197
27 107960.611648
28 97356.0681559
29 87958.3176038
30 79609.5761746
31 72175.0150947
32 65533.2392341
33 59587.1394059
34 54253.3131058
35 49458.8534175
36 45142.0778127
37 41247.2956203
38 37728.876939
39 34547.3730447
40 31666.6496497
41 29054.0800964
42 26682.329987
43 24524.8240831
44 22560.845561
45 20771.3040573
46 19138.4434996
47 17647.4416034
48 16284.8066439
49 15037.6775697
50 13896.0718028
51 12849.6281136
52 11889.7001695
53 11008.2712134
54 10198.5226615
55 9453.83687312
56 8768.44737327
57 8137.44552111
58 7555.83692847
59 7019.35439967
60 6524.32771457
61 6067.37183154
62 5644.35035009
63 5253.30592846
64 4891.62000831
65 4556.88805434
66 4246.8832809
67 3959.62072552
68 3693.6111081
69 3447.83871738
70 3219.90082651
71 3008.31426687
72 2811.66027117
73 2628.84451677
74 2458.82298369
75 2300.64645999
76 2153.4071278
77 2016.29390383
78 1888.54837403
79 1769.45005307
80 1658.3924623
81 1554.8348721
82 1458.26230952
83 1368.0811554
84 1283.91941676
85 1205.26855384
86 1131.76460961
87 1063.03186377
88 998.879738129
89 939.049008274
90 883.063050267
91 830.611537449
92 781.480565151
93 735.442839273
94 692.304856297
95 651.865767158
96 613.940631383
97 578.343063428
98 544.936213027
99 513.590695165
100 484.173799042
101 456.550656429
102 430.584262997
103 406.180465305
104 383.241760508
105 361.670608851
106 341.388912056
107 322.306987116
108 304.355869768
109 287.460334757
110 271.55280606
111 256.581022525
112 242.473677643
113 229.201321701
114 216.691745398
115 204.901094942
116 193.783890095
117 183.299301139
118 173.409671475
119 164.079907246
120 155.278794741
121 146.969693859
122 139.126053049
123 131.720001834
124 124.728507367
125 118.126146649
126 111.887699895
127 105.992337178
128 100.421131331
129 95.1579844783
130 90.1812408218
131 85.4759584953
132 81.0272193612
133 76.8203374748
134 72.8407573929
135 69.0754023792
136 65.5121181486
137 62.1408041812
138 58.9505398687
139 55.9299203056
140 53.0697393046
141 50.3620691066
142 47.7979091067
143 45.3692184678
144 43.0688037628
145 40.8896131641
146 38.8245013017
147 36.8671714931
148 35.0123220551
149 33.2543149238
150 31.5869876264
151 30.0060806177
152 28.5074011198
153 27.0864206106
154 25.738285803
155 24.4592924962
156 23.2460613481
157 22.095580461
158 21.0034560358
159 19.9670082568
160 18.9834690024
161 18.0500751217
162 17.1636976696
163 16.3222648318
164 15.5234880149
165 14.7650703235
166 14.044639401
167 13.3603778212
168 12.7109616962
169 12.0950911658
170 11.5083883494
171 10.9512610637
172 10.4216594311
173 9.91842649236
174 9.44018522656
175 8.98557325018
176 8.5533641256
177 8.14262138685
178 7.75220142684
179 7.38084619455
180 7.02774181277
181 6.69210006141
182 6.37278295152
183 6.06905994316
184 5.78024994227
185 5.50545854423
186 5.24412059909
187 4.99544784297
188 4.7587789191
189 4.53360218902
190 4.31943363815
191 4.11549310853
192 3.92141789296
193 3.73675783428
194 3.56098213041
195 3.39359387864
196 3.23428601595
197 3.08256645065
198 2.93816425021
199 2.80066562047
200 2.66968872472
201 2.5449892527
202 2.42629595759
203 2.31315864615
204 2.20542387514
205 2.10279898188
206 2.00505488908
207 1.91193790032
208 1.82322016513
209 1.73869033487
210 1.65820087617
211 1.58146427135
212 1.50834646596
213 1.43865558637
214 1.37226385322
215 1.30897770734
216 1.24864672359
217 1.19115788205
218 1.13637711414
219 1.08413447988
220 1.03434360423
221 0.986861347534
222 0.941731519118
223 0.898600887505
224 0.857442244119
225 0.818211899054
226 0.780805349589
227 0.745126838968
228 0.711096838171
229 0.678648337444
230 0.647710641224
231 0.618189844795
232 0.590039776321
233 0.563185936526
234 0.537583442341
235 0.513150237653
236 0.489840760528
237 0.467603914622
238 0.446397864368
239 0.42616172456
240 0.406848747667
241 0.388427191306
242 0.370855558425
243 0.354083342875
244 0.338077099301
245 0.322803827942
246 0.308231433716
247 0.294322889343
248 0.281047650794
249 0.268377564948
250 0.256290580567
251 0.24475181624
252 0.233735426544
253 0.223221349922
254 0.21318855445
255 0.203607414283
256 0.194462750406
257 0.185732136552
258 0.177400516897
259 0.169443820124
260 0.161849361824
261 0.154596047655
262 0.147675386228
263 0.141063647115
264 0.134750962298
265 0.128723816455
266 0.122971062483
267 0.117475104532
268 0.112227474854
269 0.107217201749
270 0.102433481398
271 0.0978638964895
272 0.0934990355514
273 0.0893308562702
274 0.0853518814327
275 0.0815613748837
276 0.0779314444591
277 0.0744637177802
278 0.0711516005017
279 0.0679869746834
280 0.0649643291142
281 0.0620768194793
282 0.0593201032657
283 0.056685644827
284 0.054169018524
285 0.051765034493
286 0.0494698746184
287 0.0472755751604
288 0.0451795412901
289 0.0431771921213
290 0.0412647645053
291 0.0394372266207
292 0.0376910135473
293 0.036022588057
294 0.0344292986833
295 0.0329065162054
296 0.0314512310268
297 0.0300606504047
298 0.0287327234898
299 0.0274631740383
300 0.0262500773169
301 0.0250909789465
302 0.0239837327717
303 0.0229252223393
304 0.0219139081886
305 0.0209473219722
306 0.020024001037
307 0.0191412115009
308 0.0182975952712
309 0.0174913799586
310 0.0167212008862
311 0.0159848020497
312 0.015280943652
313 0.0146083593546
314 0.0139659060088
315 0.0133514197323
316 0.0127641530546
317 0.0122028637876
318 0.011666637186
319 0.0111538266507
320 0.010663651593
321 0.0101951200034
322 0.0097475085077
323 0.00931955406209
324 0.00891036970299
325 0.00851923377881
326 0.00814556944464
327 0.00778816679857
328 0.00744651076696
329 0.00712022130839
330 0.00680888442279
331 0.00651043197869
332 0.0062250696326
333 0.00595224952276
334 0.0056915742786
335 0.00544229042662
336 0.00520390790625
337 0.00497603251145
338 0.00475827231446
339 0.00454997076154
340 0.00435089405885
341 0.00416051973025
342 0.00397857991577
343 0.00380455878507
344 0.00363818012722
345 0.00347910005168
346 0.00332705830751
347 0.00318163125244
348 0.00304257949394
349 0.00290963249597
350 0.00278260127194
351 0.00266106001527
352 0.00254483930922
353 0.00243371054391
354 0.00232749957138
355 0.00222590239025
356 0.0021287395606
357 0.00203583649237
358 0.001947041802
359 0.0018621202904
360 0.00178090514581
361 0.00170323206664
362 0.0016289839134
363 0.00155796568805
364 0.00149004183423
365 0.0014250935799
366 0.00136300217532
367 0.00130361090451
368 0.0012468162572
369 0.00119251454503
370 0.0011405855586
371 0.00109091379387
372 0.00104340400332
373 0.000997974317642
374 0.000954536041133
375 0.000912992190734
376 0.000873250915151
377 0.000835250206286
378 0.00079891842112
379 0.000764170334848
380 0.000730924130639
381 0.000699128169181
382 0.000668726846148
383 0.000639652882652
384 0.000611836790238
385 0.000585237197401
386 0.000559797875579
387 0.000535482126699
388 0.000512288742968
389 0.000490013188735
390 0.000468721933423
391 0.000448362105828
392 0.000428882555011
393 0.000410252538422
394 0.000392434735549
395 0.000375398037971
396 0.000359099016918
397 0.0003435047969
398 0.00032859074179
399 0.000314331882731
400 0.000300685624569
401 0.00028763466628
402 0.000275150800015
403 0.000263216535975
404 0.000251794062169
405 0.000240870845691
406 0.000230421189485
407 0.00022042960695
408 0.000210868608463
409 0.000201722384837
410 0.000192973844472
411 0.000184608466797
412 0.000176604670101
413 0.000168947178777
414 0.000161622684721
415 0.000154620492202
416 0.000147919068754
417 0.000141507629275
418 0.000135375189243
419 0.000129510105003
420 0.000123899057135
421 0.000118530392304
422 0.00011339500804
423 0.00010848383907
424 0.000103786045736
425 9.92913001532e-05
426 9.4990815559e-05
427 9.08776908734e-05
428 8.69433415523e-05
429 8.31784294012e-05
430 7.95765836457e-05
431 7.61316436427e-05
432 7.28370613798e-05
433 6.96839429268e-05
434 6.66685554027e-05
435 6.37830429815e-05
436 6.10237314766e-05
437 5.83826552461e-05
438 5.58560764084e-05
439 5.34390810128e-05
440 5.11279631803e-05
441 4.89157738116e-05
442 4.67997498816e-05
443 4.47753941553e-05
444 4.28398403209e-05
445 4.09868242134e-05
446 3.92139876691e-05
447 3.75179386517e-05
448 3.58989138568e-05
449 3.43485086433e-05
450 3.28633655062e-05
451 3.14425153211e-05
452 3.00838353239e-05
453 2.87834324339e-05
454 2.75390678223e-05
455 2.63485153923e-05
456 2.52098848127e-05
457 2.41204385818e-05
458 2.30779776418e-05
459 2.20805585323e-05
460 2.1126506491e-05
461 2.02141469491e-05
462 1.93406949957e-05
463 1.85049805141e-05
464 1.77055228767e-05
465 1.6940881147e-05
466 1.62090227097e-05
467 1.55087698078e-05
468 1.48388344548e-05
469 1.41981687965e-05
470 1.35849898236e-05
471 1.29983382342e-05
472 1.24369517112e-05
473 1.19001295801e-05
474 1.13862762021e-05
475 1.08945741878e-05
476 1.04241195847e-05
477 9.97422789474e-06
478 9.54360074158e-06
479 9.13155294037e-06
480 8.7374449629e-06
481 8.36037645584e-06
482 7.99956358303e-06
483 7.65428318781e-06
484 7.32388369921e-06
485 7.00784401443e-06
486 6.70545886926e-06
487 6.41605110938e-06
488 6.13915368264e-06
489 5.87431350344e-06
490 5.62096977936e-06
491 5.37840440236e-06
492 5.14632445292e-06
493 4.92429967763e-06
494 4.71194736299e-06
495 4.50864979199e-06
496 4.31413017081e-06
497 4.12801456623e-06
498 3.95005292715e-06
499 3.77970946178e-06