A support vector classification machine with the RBF Kernel (C=1 and gamma=0.001) was built here. And two sets of image data were tested with the model.
In [6]:
import datetime
import gc
import numpy as np
import os
import random
from scipy import misc
import string
import time
import sys
import sklearn.metrics as skm
import collections
from sklearn.svm import SVC
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot as plt
from sklearn import metrics
import dwdii_bc_model_helper as bc
random.seed(20275)
np.set_printoptions(precision=2)
In [13]:
imagePath = "png"
trainImagePath = imagePath
trainDataPath = "data/ddsm_train.csv"
testDataPath = "data/ddsm_test.csv"
categories = bc.bcNumerics()
imgResize = (150, 150)
normalVsAbnormal=False
In [14]:
os.listdir('data')
Out[14]:
In [15]:
metaData, meta2, mCounts = bc.load_training_metadata(trainDataPath, balanceViaRemoval=True, verbose=True,
normalVsAbnormal=normalVsAbnormal)
In [16]:
thesePathos = ['benign','malignant']
In [17]:
# Actually load some representative data for model experimentation
maxData = len(metaData)
X_data, Y_data = bc.load_data(trainDataPath, trainImagePath,
categories=categories,
maxData = maxData,
verboseFreq = 50,
imgResize=imgResize,
thesePathos=thesePathos,
normalVsAbnormal=normalVsAbnormal)
print X_data.shape
print Y_data.shape
In [18]:
# Actually load some representative data for model experimentation
maxData = len(metaData)
X_test, Y_test = bc.load_data(testDataPath, imagePath,
categories=categories,
maxData = maxData,
verboseFreq = 50,
imgResize=imgResize,
thesePathos=thesePathos,
normalVsAbnormal=normalVsAbnormal)
print X_test.shape
print Y_test.shape
In [19]:
X_train = X_data
Y_train = Y_data
In [20]:
print X_train.shape
print X_test.shape
print Y_train.shape
print Y_test.shape
In [21]:
def yDist(y):
bcCounts = collections.defaultdict(int)
for a in range(0, y.shape[0]):
bcCounts[y[a][0]] += 1
return bcCounts
print "Y_train Dist: " + str(yDist(Y_train))
print "Y_test Dist: " + str(yDist(Y_test))
In [22]:
X_train_s = X_train.reshape((1270,-1))
In [23]:
X_test_s = X_test.reshape((321,-1))
In [24]:
Y_train_s = Y_train.ravel()
In [30]:
model = SVC(C=1.0, gamma=0.001, kernel='rbf')
In [31]:
model.fit(X_train_s,Y_train_s)
Out[31]:
In [32]:
predicted = model.predict(X_test_s)
expected = Y_test
In [33]:
svm_matrix = skm.confusion_matrix(Y_test, predicted)
svm_matrix
Out[33]:
In [34]:
print metrics.accuracy_score(expected,predicted)
In [42]:
numBC = bc.reverseDict(categories)
class_names = numBC.values()
np.set_printoptions(precision=2)
In [43]:
# Plot non-normalized confusion matrix
plt.figure()
bc.plot_confusion_matrix(svm_matrix, classes=class_names[1:3],
title='Confusion Matrix without normalization')
plt.savefig('raw_class2_bVm_o_norm.png')
In [44]:
from IPython.display import Image
Image(filename='raw_class2_bVm_o_norm.png')
Out[44]:
In [45]:
plt.figure()
bc.plot_confusion_matrix(svm_matrix, classes=class_names[1:3], normalize=True,
title='Confusion Matrix with normalization')
plt.savefig('raw_class2_bVm_norm.png')
In [46]:
# Load the image we just saved
from IPython.display import Image
Image(filename='raw_class2_bVm_norm.png')
Out[46]:
In [47]:
imagePath = "DDSM_threshold"
trainImagePath = imagePath
trainDataPath = "data/ddsm_train.csv"
testDataPath = "data/ddsm_test.csv"
categories = bc.bcNumerics()
imgResize = (150, 150)
normalVsAbnormal=False
In [48]:
os.listdir('data')
Out[48]:
In [49]:
metaData, meta2, mCounts = bc.load_training_metadata(trainDataPath, balanceViaRemoval=True, verbose=True,
normalVsAbnormal=normalVsAbnormal)
In [50]:
# Actually load some representative data for model experimentation
maxData = len(metaData)
X_data, Y_data = bc.load_data(trainDataPath, trainImagePath,
categories=categories,
maxData = maxData,
verboseFreq = 50,
imgResize=imgResize,
thesePathos=thesePathos,
normalVsAbnormal=normalVsAbnormal)
print X_data.shape
print Y_data.shape
In [51]:
# Actually load some representative data for model experimentation
maxData = len(metaData)
X_test, Y_test = bc.load_data(testDataPath, imagePath,
categories=categories,
maxData = maxData,
verboseFreq = 50,
imgResize=imgResize,
thesePathos=thesePathos,
normalVsAbnormal=normalVsAbnormal)
print X_test.shape
print Y_test.shape
In [52]:
X_train = X_data
Y_train = Y_data
In [53]:
print X_train.shape
print X_test.shape
print Y_train.shape
print Y_test.shape
In [54]:
def yDist(y):
bcCounts = collections.defaultdict(int)
for a in range(0, y.shape[0]):
bcCounts[y[a][0]] += 1
return bcCounts
print "Y_train Dist: " + str(yDist(Y_train))
print "Y_test Dist: " + str(yDist(Y_test))
In [55]:
X_train_s = X_train.reshape((1196,-1))
X_test_s = X_test.reshape((309,-1))
Y_train_s = Y_train.ravel()
In [56]:
model = SVC(C=1.0, gamma=0.001, kernel='rbf')
model.fit(X_train_s,Y_train_s)
Out[56]:
In [57]:
predicted = model.predict(X_test_s)
expected = Y_test
In [58]:
svm_matrix = skm.confusion_matrix(Y_test, predicted)
svm_matrix
Out[58]:
In [59]:
print metrics.accuracy_score(expected,predicted)
In [60]:
numBC = bc.reverseDict(categories)
class_names = numBC.values()
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
plt.figure()
bc.plot_confusion_matrix(svm_matrix, classes=class_names[1:3],
title='Confusion Matrix without normalization')
plt.savefig('threshold_class2_bVm_o_norm.png')
In [61]:
from IPython.display import Image
Image(filename='threshold_class2_bVm_o_norm.png')
Out[61]:
In [62]:
plt.figure()
bc.plot_confusion_matrix(svm_matrix, classes=class_names[1:3], normalize=True,
title='Confusion Matrix with normalization')
plt.savefig('threshold_class2_bVm_norm.png')
In [63]:
# Load the image we just saved
from IPython.display import Image
Image(filename='threshold_class2_bVm_norm.png')
Out[63]:
In [ ]: