https://youtu.be/FvamL5oA_EE?list=PLQVvvaa0QuDc-3szzjeP6N6b0aDrrKyL-
In [1]:
import quandl;
import pandas as pd;
import pickle;
import matplotlib.pyplot as plt;
from matplotlib import style;
style.use("ggplot");
In [9]:
api_key = open("quandlapikey.txt", "r").read();
def mortgage_30y():
df = quandl.get("FMAC/MORTG", trim_start = "1975-01-01", authtoken = api_key);
df["Value"] = (df["Value"] - df["Value"][0]) / df["Value"][0] * 100.0;
df.columns = ["M30"];
return df;
def state_list():
fiddy_states = pd.read_html("https://simple.wikipedia.org/wiki/List_of_U.S._states");
return fiddy_states[0][0][1:];
def grap_initial_state_data_start_pct():
states = state_list();
main_df = pd.DataFrame();
for ab in states:
querry = "FMAC/HPI_" + ab;
df = quandl.get(querry, authtoken = api_key);
df.columns = [ab];
df[ab] = (df[ab] - df[ab][0]) / df[ab][0] * 100.0; # <-------
if main_df.empty:
main_df = df;
else:
main_df = main_df.join(df);
pickle_out = open("./data/fiddy_states.pickle", "wb");
pickle.dump(main_df, pickle_out);
pickle_out.close();
def HPI_Benchmark():
df = quandl.get("FMAC/HPI_USA", authtoken = api_key);
df.columns = ["US"];
df["US"] = (df["US"] - df["US"][0]) / df["US"][0] * 100.0; # <-------
return df;
In [10]:
df = mortgage_30y();
print(df.head()); # begin of month
M30
Date
1975-01-01 0.000000
1975-02-01 -3.393425
1975-03-01 -5.620361
1975-04-01 -6.468717
1975-05-01 -5.514316
In [11]:
HPI_data = pd.read_pickle("./data/fiddy_states.pickle");
print(HPI_data.head()); # end of month
AL AK AZ AR CA CO \
Date
1975-01-31 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
1975-02-28 0.626500 1.485775 1.688504 0.846192 0.356177 1.159639
1975-03-31 1.358575 3.006473 3.261346 1.581956 1.575690 2.299449
1975-04-30 2.254726 4.593530 4.475810 2.183669 3.573196 3.359028
1975-05-31 3.107829 6.327600 5.139617 2.786248 5.241395 4.226895
CT DE FL GA ... SD \
Date ...
1975-01-31 0.000000 0.000000 0.000000 0.000000 ... 0.000000
1975-02-28 2.123926 0.142451 3.938796 -0.902841 ... -1.099859
1975-03-31 3.719898 0.387918 9.798243 -1.282758 ... -2.218400
1975-04-30 4.616778 0.891619 16.974819 -1.068371 ... -3.328868
1975-05-31 4.901787 1.752086 17.891884 -0.676830 ... -4.335377
TN TX UT VT VA WA \
Date
1975-01-31 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
1975-02-28 0.112132 0.951488 1.398290 0.945247 0.987288 0.397561
1975-03-31 0.369186 2.701039 2.659262 1.857227 1.707474 1.015723
1975-04-30 0.765570 5.276570 3.662119 2.685578 2.238392 1.757887
1975-05-31 1.228808 6.099286 4.326186 3.413295 2.684036 2.426439
WV WI WY
Date
1975-01-31 0.000000 0.000000 0.000000
1975-02-28 2.495069 1.404920 1.438502
1975-03-31 5.093474 2.791371 2.962512
1975-04-30 7.772685 4.034165 4.545011
1975-05-31 10.469784 5.047628 6.080901
[5 rows x 50 columns]
In [12]:
def mortgage_30y_resampled():
df = quandl.get("FMAC/MORTG", trim_start = "1975-01-01", authtoken = api_key);
df["Value"] = (df["Value"] - df["Value"][0]) / df["Value"][0] * 100.0;
df.columns = ["M30"];
return df.resample("M").mean();
m30 = mortgage_30y_resampled();
print(m30.head()); # begin of month
M30
Date
1975-01-31 0.000000
1975-02-28 -3.393425
1975-03-31 -5.620361
1975-04-30 -6.468717
1975-05-31 -5.514316
In [14]:
HPI_bench = HPI_Benchmark();
state_HPI_M30 = HPI_data.join(m30);
print(state_HPI_M30.head());
AL AK AZ AR CA CO \
Date
1975-01-31 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
1975-02-28 0.626500 1.485775 1.688504 0.846192 0.356177 1.159639
1975-03-31 1.358575 3.006473 3.261346 1.581956 1.575690 2.299449
1975-04-30 2.254726 4.593530 4.475810 2.183669 3.573196 3.359028
1975-05-31 3.107829 6.327600 5.139617 2.786248 5.241395 4.226895
CT DE FL GA ... TN \
Date ...
1975-01-31 0.000000 0.000000 0.000000 0.000000 ... 0.000000
1975-02-28 2.123926 0.142451 3.938796 -0.902841 ... 0.112132
1975-03-31 3.719898 0.387918 9.798243 -1.282758 ... 0.369186
1975-04-30 4.616778 0.891619 16.974819 -1.068371 ... 0.765570
1975-05-31 4.901787 1.752086 17.891884 -0.676830 ... 1.228808
TX UT VT VA WA WV \
Date
1975-01-31 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
1975-02-28 0.951488 1.398290 0.945247 0.987288 0.397561 2.495069
1975-03-31 2.701039 2.659262 1.857227 1.707474 1.015723 5.093474
1975-04-30 5.276570 3.662119 2.685578 2.238392 1.757887 7.772685
1975-05-31 6.099286 4.326186 3.413295 2.684036 2.426439 10.469784
WI WY M30
Date
1975-01-31 0.000000 0.000000 0.000000
1975-02-28 1.404920 1.438502 -3.393425
1975-03-31 2.791371 2.962512 -5.620361
1975-04-30 4.034165 4.545011 -6.468717
1975-05-31 5.047628 6.080901 -5.514316
[5 rows x 51 columns]
AL AK AZ AR CA CO CT \
AL 1.000000 0.946418 0.937592 0.995119 0.942310 0.965541 0.953146
AK 0.946418 1.000000 0.910237 0.967983 0.921818 0.960382 0.884600
AZ 0.937592 0.910237 1.000000 0.936454 0.976776 0.919486 0.917688
AR 0.995119 0.967983 0.936454 1.000000 0.945774 0.976929 0.944746
CA 0.942310 0.921818 0.976776 0.945774 1.000000 0.938870 0.942463
CO 0.965541 0.960382 0.919486 0.976929 0.938870 1.000000 0.889392
CT 0.953146 0.884600 0.917688 0.944746 0.942463 0.889392 1.000000
DE 0.982824 0.938068 0.948801 0.977921 0.967911 0.936764 0.979810
FL 0.929395 0.900713 0.994380 0.928081 0.985869 0.910895 0.926355
GA 0.978346 0.899525 0.949167 0.968410 0.951573 0.956025 0.947469
HI 0.944752 0.947413 0.916923 0.951617 0.951263 0.928241 0.915631
ID 0.983136 0.953404 0.965198 0.980867 0.954768 0.962413 0.917955
IL 0.983692 0.899603 0.947529 0.971113 0.953901 0.939419 0.958374
IN 0.983855 0.903966 0.899364 0.977312 0.914634 0.961496 0.922407
IA 0.985434 0.954436 0.912447 0.989720 0.930562 0.985693 0.910157
KS 0.989496 0.964757 0.926154 0.995277 0.941173 0.989344 0.928693
KY 0.994332 0.946701 0.917390 0.994178 0.936200 0.978925 0.937581
LA 0.966664 0.992502 0.924051 0.982658 0.930218 0.975262 0.897732
ME 0.976759 0.940709 0.936183 0.976928 0.968159 0.946687 0.983722
MD 0.969675 0.944668 0.965352 0.969750 0.980935 0.933739 0.967394
MA 0.970801 0.930523 0.934268 0.972517 0.966969 0.955733 0.978691
MI 0.903473 0.771111 0.863715 0.886418 0.870084 0.888848 0.858025
MN 0.976315 0.934883 0.959047 0.977211 0.972211 0.972480 0.944407
MS 0.996520 0.963309 0.938513 0.997697 0.941894 0.973874 0.944285
MO 0.993873 0.938797 0.947188 0.991297 0.958757 0.972391 0.959064
MT 0.974121 0.982212 0.928209 0.983787 0.939705 0.975017 0.905319
NE 0.986508 0.953399 0.914148 0.991408 0.930202 0.988325 0.913586
NV 0.872598 0.801393 0.964619 0.861622 0.943090 0.838751 0.876240
NH 0.959618 0.913121 0.940935 0.958676 0.967608 0.932572 0.987413
NJ 0.971680 0.928850 0.948086 0.968597 0.968653 0.927007 0.989316
NM 0.993101 0.962755 0.943021 0.992937 0.939615 0.953317 0.947403
NY 0.977840 0.944144 0.932909 0.978121 0.962826 0.944382 0.983996
NC 0.998317 0.941936 0.929046 0.993249 0.939050 0.968365 0.952985
ND 0.902227 0.964140 0.846203 0.929000 0.883047 0.950659 0.823583
OH 0.969046 0.862634 0.894522 0.956087 0.905167 0.936379 0.918310
OK 0.947181 0.988448 0.901707 0.969293 0.908747 0.969076 0.871562
OR 0.977741 0.959812 0.955111 0.979389 0.956000 0.975643 0.905014
PA 0.987218 0.951013 0.928817 0.986576 0.956387 0.951410 0.969914
RI 0.954402 0.895370 0.946265 0.948688 0.974351 0.910224 0.988253
SC 0.998306 0.951217 0.941651 0.994964 0.951189 0.972161 0.955383
SD 0.976382 0.974957 0.906742 0.986937 0.930504 0.986830 0.905662
TN 0.994551 0.958200 0.932276 0.995595 0.945515 0.982027 0.940744
TX 0.940841 0.980514 0.896961 0.962400 0.917953 0.977895 0.873570
UT 0.979483 0.963944 0.936448 0.982990 0.928609 0.978576 0.890769
VT 0.978591 0.956485 0.924615 0.981747 0.956815 0.948834 0.967854
VA 0.975277 0.962226 0.958568 0.979944 0.979699 0.954017 0.962382
WA 0.985060 0.958805 0.956881 0.984041 0.965064 0.972331 0.929065
WV 0.982097 0.972376 0.919514 0.990395 0.936562 0.977987 0.914855
WI 0.991161 0.930831 0.935717 0.986244 0.945139 0.967228 0.939192
WY 0.946695 0.987373 0.909381 0.964090 0.919537 0.959549 0.871945
M30 -0.796453 -0.734748 -0.673961 -0.788178 -0.733805 -0.787303 -0.750397
DE FL GA ... TN TX UT \
AL 0.982824 0.929395 0.978346 ... 0.994551 0.940841 0.979483
AK 0.938068 0.900713 0.899525 ... 0.958200 0.980514 0.963944
AZ 0.948801 0.994380 0.949167 ... 0.932276 0.896961 0.936448
AR 0.977921 0.928081 0.968410 ... 0.995595 0.962400 0.982990
CA 0.967911 0.985869 0.951573 ... 0.945515 0.917953 0.928609
CO 0.936764 0.910895 0.956025 ... 0.982027 0.977895 0.978576
CT 0.979810 0.926355 0.947469 ... 0.940744 0.873570 0.890769
DE 1.000000 0.953177 0.960868 ... 0.974580 0.918837 0.950357
FL 0.953177 1.000000 0.944083 ... 0.922868 0.882969 0.918716
GA 0.960868 0.944083 1.000000 ... 0.974288 0.915710 0.949785
HI 0.964760 0.918086 0.904039 ... 0.953317 0.937485 0.943980
ID 0.971288 0.954029 0.960638 ... 0.980437 0.941788 0.990016
IL 0.979059 0.950143 0.983955 ... 0.969193 0.888752 0.947679
IN 0.949262 0.891750 0.977512 ... 0.982635 0.917451 0.958129
IA 0.957725 0.904738 0.957605 ... 0.990240 0.954038 0.983645
KS 0.965136 0.917737 0.968449 ... 0.994828 0.968453 0.983539
KY 0.970905 0.911063 0.973330 ... 0.996153 0.949643 0.977698
LA 0.949326 0.911729 0.925457 ... 0.975008 0.981885 0.981256
ME 0.993703 0.942408 0.959319 ... 0.975749 0.931723 0.942646
MD 0.994339 0.971156 0.947545 ... 0.963179 0.918259 0.947790
MA 0.980283 0.940686 0.970928 ... 0.972890 0.933591 0.934550
MI 0.864550 0.862790 0.954318 ... 0.896473 0.801939 0.861986
MN 0.970738 0.961250 0.980903 ... 0.974000 0.931768 0.958771
MS 0.978320 0.928466 0.970387 ... 0.994905 0.955900 0.985066
MO 0.980589 0.944549 0.988700 ... 0.990258 0.940030 0.968993
MT 0.963541 0.919298 0.929777 ... 0.981426 0.967914 0.989110
NE 0.953909 0.904769 0.968433 ... 0.993074 0.961815 0.980702
NV 0.892577 0.972301 0.918296 ... 0.856773 0.790516 0.848739
NH 0.978789 0.949645 0.964654 ... 0.956148 0.910975 0.915630
NJ 0.995439 0.955855 0.956171 ... 0.962878 0.908840 0.932115
NM 0.982960 0.933961 0.957731 ... 0.985988 0.943133 0.979742
NY 0.993968 0.937888 0.957303 ... 0.976450 0.933915 0.943333
NC 0.979568 0.920931 0.980970 ... 0.996425 0.944008 0.976614
ND 0.887317 0.832715 0.849939 ... 0.934301 0.974275 0.941140
OH 0.935233 0.890929 0.979829 ... 0.960554 0.876447 0.929879
OK 0.921562 0.886867 0.910634 ... 0.959073 0.986802 0.964156
OR 0.964901 0.946589 0.952781 ... 0.981804 0.951085 0.992893
PA 0.994418 0.929922 0.955997 ... 0.985517 0.939522 0.960547
RI 0.985562 0.961259 0.956066 ... 0.944553 0.880873 0.905477
SC 0.984438 0.935053 0.981719 ... 0.996202 0.949965 0.979195
SD 0.956217 0.897824 0.939083 ... 0.988998 0.975858 0.984643
TN 0.974580 0.922868 0.974288 ... 1.000000 0.963608 0.985872
TX 0.918837 0.882969 0.915710 ... 0.963608 1.000000 0.961318
UT 0.950357 0.918716 0.949785 ... 0.985872 0.961318 1.000000
VT 0.991994 0.927173 0.942940 ... 0.979467 0.942039 0.954472
VA 0.992477 0.961733 0.949727 ... 0.975512 0.943471 0.960229
WA 0.979299 0.951094 0.962595 ... 0.986922 0.952730 0.987649
WV 0.965557 0.910600 0.940694 ... 0.988203 0.963840 0.986347
WI 0.974029 0.934246 0.977461 ... 0.984005 0.921060 0.970967
WY 0.938417 0.898932 0.891159 ... 0.956694 0.966035 0.974129
M30 -0.790202 -0.682449 -0.752580 ... -0.808287 -0.726491 -0.782244
VT VA WA WV WI WY M30
AL 0.978591 0.975277 0.985060 0.982097 0.991161 0.946695 -0.796453
AK 0.956485 0.962226 0.958805 0.972376 0.930831 0.987373 -0.734748
AZ 0.924615 0.958568 0.956881 0.919514 0.935717 0.909381 -0.673961
AR 0.981747 0.979944 0.984041 0.990395 0.986244 0.964090 -0.788178
CA 0.956815 0.979699 0.965064 0.936562 0.945139 0.919537 -0.733805
CO 0.948834 0.954017 0.972331 0.977987 0.967228 0.959549 -0.787303
CT 0.967854 0.962382 0.929065 0.914855 0.939192 0.871945 -0.750397
DE 0.991994 0.992477 0.979299 0.965557 0.974029 0.938417 -0.790202
FL 0.927173 0.961733 0.951094 0.910600 0.934246 0.898932 -0.682449
GA 0.942940 0.949727 0.962595 0.940694 0.977461 0.891159 -0.752580
HI 0.975666 0.975532 0.970579 0.960787 0.925968 0.951080 -0.795357
ID 0.963029 0.974919 0.994989 0.977901 0.976381 0.963630 -0.767660
IL 0.958690 0.964414 0.969529 0.951565 0.991507 0.904993 -0.779350
IN 0.947160 0.938717 0.958916 0.962436 0.984325 0.903299 -0.806090
IA 0.966561 0.963537 0.980342 0.992064 0.988385 0.962935 -0.820293
KS 0.972309 0.970785 0.981429 0.989891 0.986305 0.964479 -0.789270
KY 0.975415 0.968466 0.979889 0.986531 0.991031 0.947108 -0.818853
LA 0.963140 0.967926 0.974058 0.986051 0.955028 0.991333 -0.737043
ME 0.994193 0.991146 0.970766 0.964921 0.968257 0.935928 -0.802695
MD 0.986123 0.996590 0.978610 0.962117 0.964728 0.947355 -0.768376
MA 0.979159 0.979110 0.959213 0.953810 0.967552 0.918743 -0.795487
MI 0.838633 0.846263 0.871264 0.854135 0.927605 0.767608 -0.722214
MN 0.960967 0.974144 0.974422 0.963952 0.988026 0.935965 -0.765481
MS 0.979034 0.977545 0.985463 0.988062 0.987715 0.962575 -0.790514
MO 0.973695 0.975935 0.980125 0.974391 0.993836 0.936164 -0.784704
MT 0.975319 0.978356 0.988061 0.994438 0.966600 0.992572 -0.798324
NE 0.961521 0.958325 0.975139 0.985446 0.984822 0.950928 -0.800434
NV 0.845819 0.890009 0.887098 0.829693 0.886561 0.796793 -0.596116
NH 0.970435 0.974311 0.947221 0.934251 0.956025 0.901631 -0.760839
NJ 0.986350 0.989094 0.964363 0.951430 0.964999 0.925147 -0.774544
NM 0.979584 0.979834 0.983676 0.981711 0.979327 0.962241 -0.766544
NY 0.994886 0.990348 0.969834 0.964604 0.965848 0.936187 -0.804832
NC 0.977722 0.971135 0.982096 0.979189 0.987937 0.939552 -0.805513
ND 0.927418 0.923280 0.929351 0.955674 0.882989 0.965009 -0.787380
OH 0.921110 0.916725 0.936785 0.931484 0.977248 0.860133 -0.781626
OK 0.939188 0.943892 0.950984 0.968331 0.933125 0.977162 -0.689749
OR 0.963832 0.975474 0.996327 0.985248 0.976817 0.974578 -0.791154
PA 0.997804 0.990727 0.981644 0.979808 0.974825 0.950632 -0.812888
RI 0.969925 0.976730 0.948184 0.926765 0.954967 0.890629 -0.763330
SC 0.981423 0.979027 0.987777 0.982225 0.989545 0.949415 -0.791708
SD 0.974324 0.969555 0.980315 0.994590 0.969068 0.978354 -0.827213
TN 0.979467 0.975512 0.986922 0.988203 0.984005 0.956694 -0.808287
TX 0.942039 0.943471 0.952730 0.963840 0.921060 0.966035 -0.726491
UT 0.954472 0.960229 0.987649 0.986347 0.970967 0.974129 -0.782244
VT 1.000000 0.992917 0.976825 0.978471 0.965785 0.956335 -0.812577
VA 0.992917 1.000000 0.984165 0.976119 0.968290 0.962492 -0.786206
WA 0.976825 0.984165 1.000000 0.985030 0.979435 0.969017 -0.791745
WV 0.978471 0.976119 0.985030 1.000000 0.976364 0.981313 -0.810091
WI 0.965785 0.968290 0.979435 0.976364 1.000000 0.938097 -0.806441
WY 0.956335 0.962492 0.969017 0.981313 0.938097 1.000000 -0.753197
M30 -0.812577 -0.786206 -0.791745 -0.810091 -0.806441 -0.753197 1.000000
[51 rows x 51 columns]
In [15]:
state_HPI_M30.corr().plot();
plt.show();
In [16]:
print(state_HPI_M30.corr()["M30"].describe());
# M30 correlates with HPI
count 51.000000
mean -0.738489
std 0.251952
min -0.827213
25% -0.799379
50% -0.787303
75% -0.757018
max 1.000000
Name: M30, dtype: float64
In [ ]:
Content source: shevkunov/workout
Similar notebooks: