In [1]:
%matplotlib inline
%pylab inline
In [2]:
import warnings
warnings.filterwarnings('ignore')
In [3]:
import numpy as np
import matplotlib.pyplot as plt
from sklearn import preprocessing
import seaborn as sns
import pandas as pd
In [4]:
IMG_DIR = '../../analysis'
DPI=120
In [5]:
s = 10000 # desired sample size
n = 5967780
%time rows_to_skip = sorted(np.random.choice(np.arange(1, n + 1), (n - s), replace=False))
len(rows_to_skip)
Out[5]:
In [6]:
%time df = pd.read_csv('../../data/2001.csv', encoding='iso-8859-1', engine='c', skiprows=rows_to_skip)
len(df)
Out[6]:
In [7]:
df.head()
Out[7]:
In [8]:
# http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelEncoder.html
# turn those text labels into numerical
text_cols = ['UniqueCarrier', 'Origin', 'Dest']
le = preprocessing.LabelEncoder()
for c in text_cols:
# print (c,set(df[c].values))
flist = list(set(df[c].values))
# print(flist)
le.fit(flist)
leo = le.transform(flist)
# print (c,flist,leo)
df[c+'_'] = df[c]
df[c+'_'].replace(flist,value=leo,inplace=True)
In [9]:
df.head()
Out[9]:
In [10]:
df.fillna(-1, inplace=True)
In [11]:
df.head()
Out[11]:
In [12]:
cols_for_correlation = [
'DayOfWeek',
'DepTime',
'ArrTime',
'ArrDelay',
'Distance',
'UniqueCarrier_',
'Origin_',
'Dest_'
]
In [13]:
corrmat = df[cols_for_correlation].corr()
sns.heatmap(corrmat, annot=True)
figure = plt.gcf()
figure.set_size_inches(10, 10)
# plt.show()
plt.savefig(IMG_DIR+'/corr.png', dpi = DPI)
In [14]:
def plot(col1, col2):
# https://stanford.edu/~mwaskom/software/seaborn/generated/seaborn.jointplot.html#seaborn.jointplot
sns.jointplot(df[col1],df[col2],dropna=True, kind="hex")
figure = plt.gcf()
figure.set_size_inches(10, 10)
# for notebook
# plt.show()
plt.savefig('%s/%s_%s.png'%(IMG_DIR, col1, col2), dpi = DPI)
In [15]:
plot('ArrTime', 'DepTime')
In [16]:
plot('Distance', 'UniqueCarrier_')
In [17]:
plot('Origin_', 'UniqueCarrier_')
In [ ]:
In [ ]: