In [2458]:
%matplotlib inline
import numpy as np
import pandas as pd
import scipy
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
import math
from matplotlib.mlab import PCA as mlabPCA
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn import preprocessing
from sklearn.feature_selection import SelectKBest
import seaborn as sns
import scipy.stats as stats
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.naive_bayes import BernoulliNB
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import cross_val_score, KFold
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification
from sklearn.feature_selection import RFE
from sklearn.model_selection import cross_val_predict
from sklearn import metrics
from sklearn.decomposition import PCA as sklearn_pca
import locale
from locale import atof
import warnings
from IPython.display import display
from sklearn import linear_model
In [2459]:
# Import FBI Raw Data
fbidata = pd.read_csv('https://raw.githubusercontent.com/Thinkful-Ed/data-201-resources/master/New_York_offenses/NEW_YORK-Offenses_Known_to_Law_Enforcement_by_City_2013%20-%2013tbl8ny.csv', delimiter=",", thousands=',',decimal=".")
fbiraw = pd.DataFrame(fbidata)
fbiraw.head()
Out[2459]:
In [2460]:
#Transform FBI Raw Data
#Rename columns with row 3 from the original data set
fbiraw_t1 = fbiraw.rename(columns=fbiraw.iloc[3])
#Delete first three rows don´t contain data for the regression model
fbiraw_t2 = fbiraw_t1.drop(fbiraw_t1.index[0:4])
In [2461]:
#Delete column "Rape (revised definition)1 as it contains no data
fbiraw_t2 = fbiraw_t2.drop('Rape\n(revised\ndefinition)1', axis = 1)
In [2462]:
#Delete Arson Column as there is insufficient data
# 'The FBI does not publish arson data unless it receives data from either the agency or the state
# for all 12 months of the calendar year.'
fbiraw_t2 = fbiraw_t2.drop('Arson3', axis = 1)
In [2463]:
#Clean tail from the data set
#Re-shape dataset excluding the last 3 rows of the dataset as they don´t contain relevant information for the model
fbiraw_t2 = fbiraw_t2[:-3]
#Change names in Columns
fbiraw_t2= fbiraw_t2.rename(columns={'Violent\ncrime': 'Violent Crime', 'Murder and\nnonnegligent\nmanslaughter': 'Murder','Rape\n(legacy\ndefinition)2': 'Rape', 'Robbery': 'Robbery', 'Aggravated\nassault': 'Assault', 'Property\ncrime': 'PropertyCrime', 'Burglary': 'Burglary', 'Larceny-\ntheft': 'Larceny & Theft', 'Motor\nvehicle\ntheft': 'Motor Vehicle Theft'})
In [2464]:
#Analyse missing information
fbiraw_t2.info()
In [2465]:
#Change all columns from object to float
locale.setlocale(locale.LC_NUMERIC, '')
fbiraw_t2['Population'] = fbiraw_t2['Population'].apply(atof)
fbiraw_t2['Violent Crime'] = fbiraw_t2['Violent Crime'].apply(atof)
fbiraw_t2['Murder'] = fbiraw_t2['Murder'].apply(atof)
fbiraw_t2['Rape'] = fbiraw_t2['Rape'].apply(atof)
fbiraw_t2['Robbery'] = fbiraw_t2['Robbery'].apply(atof)
fbiraw_t2['Assault'] = fbiraw_t2['Assault'].apply(atof)
fbiraw_t2['PropertyCrime'] = fbiraw_t2['PropertyCrime'].apply(atof)
fbiraw_t2['Burglary'] = fbiraw_t2['Burglary'].apply(atof)
fbiraw_t2['Larceny & Theft'] = fbiraw_t2['Larceny & Theft'].apply(atof)
fbiraw_t2['Motor Vehicle Theft'] = fbiraw_t2['Motor Vehicle Theft'].apply(atof)
fbiraw_t2.info()
In [2466]:
#Reindex the dataframe
fbiraw_t3 = fbiraw_t2.reset_index(drop=True)
In [2467]:
#Extract only the columns that are needed
fbiraw_t3 = fbiraw_t3[['City','PropertyCrime','Population','Murder','Robbery']]
In [2468]:
#Eliminate outliers
fbiraw_t3 = fbiraw_t3[fbiraw_t3.PropertyCrime < 170].reset_index(drop=True)
#Describe the dataset
fbiraw_t3.describe()
Out[2468]:
In [2469]:
#Print length of dataset and sort values by Population to see how many datapoints are excluded
print(len(fbiraw_t3), len(fbiraw_t2) - len(fbiraw_t3))
fbiraw_t3.sort_values('Population',ascending=False).head()
Out[2469]:
In [2470]:
#Plot the relationships between variables
sns.set_style("white")
#Conisder only the vairables suitable for the model
dfcont = fbiraw_t3[['PropertyCrime','Population','Murder','Robbery']]
# Scatterplot matrix.
g = sns.PairGrid(dfcont, diag_sharey=False)
g.map_upper(plt.scatter, alpha=.5)
# Fit line summarizing the linear relationship of the two variables.
g.map_lower(sns.regplot, scatter_kws=dict(alpha=0))
# Give information about the univariate distributions of the variables.
g.map_diag(sns.kdeplot, lw=3)
plt.show()
In [2471]:
# Initialize the figure with a logarithmic x axis
f, ax = plt.subplots(figsize=(7, 6))
ax.set_xscale("log")
# Define the variables that are going to be plot
df_long = fbiraw_t3[['PropertyCrime', 'Population']]
#Boxplot vairables
ax = sns.boxplot(data=df_long, orient="h", palette="Set2")
In [2472]:
#Create the new feature Population2
fbiraw_t3['Population2'] = fbiraw_t3['Population']*fbiraw_t3['Population']
In [2473]:
#Convert Robbery into a categorical feature
fbiraw_t3.loc[fbiraw_t3['Robbery'] > 0, 'Robbery'] = 1
In [2474]:
#Convert Murder into a categorical feature
fbiraw_t3.loc[fbiraw_t3['Murder'] > 0, 'Murder'] = 1
In [2475]:
#Transform dataset into final dataset with features
fbidata = fbiraw_t3[['PropertyCrime','Population', 'Population2','Murder','Robbery']]
fbidata.head()
Out[2475]:
In [2476]:
# Instantiate and fit our model.
regr = linear_model.LinearRegression()
Y = fbidata['PropertyCrime'].values.reshape(-1, 1)
X = fbidata[['Population', 'Population2','Murder','Robbery']]
regr.fit(X, Y)
# Inspect the results.
print('\nCoefficients: \n', regr.coef_)
print('\nIntercept: \n', regr.intercept_)
print('\nR-squared:')
print(regr.score(X, Y))
In [2477]:
#
outcome = fbidata['PropertyCrime']
feature = fbidata['Population']
# Plot the data as-is. Looks a mite quadratic.
plt.scatter(outcome, feature)
plt.title('Raw values')
plt.show()
In [2478]:
# Extract predicted values.
predicted = regr.predict(X).ravel()
actual = fbidata['PropertyCrime']
# Calculate the error, also called the residual.
residual = actual - predicted
# This looks a bit concerning.
plt.hist(residual)
plt.title('Residual counts')
plt.xlabel('Residual')
plt.ylabel('Count')
plt.show()
In [2479]:
plt.scatter(predicted, residual)
plt.xlabel('Predicted')
plt.ylabel('Residual')
plt.axhline(y=0)
plt.title('Residual vs. Predicted')
plt.show()
In [2480]:
correlation_matrix = X.corr()
display(correlation_matrix)