In [143]:

    

from __future__ import print_function
import random
import numpy as np
from matplotlib import pyplot as plt
import math


    

In [144]:

    

# Generate random data
def uniform_randoms(start, end, count):
    return [random.uniform(start, end) for _ in range(count)]

def generate_datasets(start, end, count, sets):
    return np.array([uniform_randoms(start, end, count) for _ in range(sets)])

# define f(x) 
def f(x, e=None):
    if e is None:
        e = random.gauss(0.0, math.sqrt(0.1))
    return 2 * x * x + e

def generate_ys(datasets):
    yss = []
    for xs in datasets:
        ys =[f(x) for x in xs]
        yss.append(ys)
    return np.array(yss)


    

In [160]:

    

def predict(X, W):
    return np.matmul(X, W)

def MSECost(Y2, Y1):
    return float(np.sum((Y2 - Y1) ** 2) / len(Y2))

def fit_line(X, Y):
    return np.matmul(
        np.matmul(
            np.linalg.pinv(np.matmul(X.transpose(), X)),
            X.transpose()),
        Y)

class LinearRegression(object):
    
    def __init__(self, X, Y):
        self.means = X.mean(axis=0)
        self.stds = X.std(axis=0)
        X = self.normalize(X)
        self.W = self._find_W(X, Y)
        #print(X.shape, self.W.shape, self.means, self.stds)

    def _find_W(self, X, Y):
        return fit_line(X, Y)

    def normalize(self, X, standardize=False):
        if standardize:
            X = (X - self.means) / self.stds
        # Bias is added as a weight to simplify the calculations
        X = np.insert(X, 0, 1, axis=1)
        return X
    
    def predict(self, X):        
        X = self.normalize(X)
        return np.matmul(X, self.W)
    
    def __repr__(self):
        return "W:%s, means:%s, Stds:%s" % (self.W, self.means, self.stds)

def phi(X, high, low=1):
    phi_X = np.zeros(shape=(len(X), 0))
    for power in range(low, high+1):
        phi_X = np.insert(phi_X, phi_X.shape[1], X ** power, axis=1)
    return phi_X

highest_poly_deg = 4
def report_bias_variance(n, num_ds, m=1000):
    print("Number of datasets=%d, Number of Samples=%d, number of samples in testset:%d" % (num_ds, n, m))
    print("Function\t Variance\t\t Bias^2")
    datasets = generate_datasets(-1, 1, n, num_ds)
    test_set = generate_datasets(-1, 1, m, 1)[0]
    yss = generate_ys(datasets)
    test_ys = generate_ys([test_set])[0]
    
    plt.figure(1, figsize=(15, 10))
    # G1 is a constant output function, no learning required, thus no regression
    predY = np.array([1.0 for _ in range(m)])
    cost = MSECost(test_ys, predY)
    costs = np.array([cost for _ in range(num_ds)]) # all of them have same cost
    variance = 0 #
    bias_2 = np.sum((predY - test_ys)**2) / m
    print("g1\t\t %.8f\t\t %.8f" % (variance, bias_2))
    plt.subplot(2, 3, 1)
    plt.hist(costs, 10, facecolor='green', alpha=0.75)
    plt.xlabel('Sum of Squared Errors')
    plt.ylabel('No of Datasets')
    plt.title(r'$g_%d(x)=1$  $\mu=%f, \sigma=%f$.' % (1, costs.mean(), costs.std()))
    
    #print("Xs:",datasets)
    for j in range(highest_poly_deg + 1): # highest power of X
        models = [LinearRegression(phi(datasets[i], j), yss[i]) for i in range(num_ds)]
        phi_test_set = phi(test_set, j)
        preds = np.zeros(shape=(num_ds, m))        
        costs = np.zeros(num_ds)
        for row in range(num_ds):
            pred_y = models[row].predict(phi_test_set)
            costs[row] = MSECost(pred_y, test_ys)
            for col in range(m):
                preds[row,col] = pred_y[col]

        avgx = preds.mean(axis=0) # averaging over datasets, axis=0 is data_sets
        spread = np.zeros(shape=(num_ds, m))
        for col in range(m):
            for row in range(num_ds):
                spread[row, col] = abs(preds[row,col] - avgx[col])
        variance = np.sum(spread ** 2) / (float(num_ds * m))
                
        bias_2 = 0.
        for col in range(m):
            bias_2 += (avgx[col] - test_ys[col]) ** 2
        bias_2 = bias_2 / m
        
        print("%s\t\t %.8f\t\t %.8f" % ("g%d" % (j+2), variance, bias_2))

        #print(costs)
        plt.subplot(2, 3, 2 + j)
        plt.hist(costs, 10, facecolor='green', alpha=0.75)
        plt.xlabel('Sum of Squared Errors')
        plt.ylabel('No of Datasets')
        plt.title(r'$g_%d(x)$  $\mu=%f, \sigma=%f$.' % (j + 2, costs.mean(), costs.std()))

    plt.show()
report_bias_variance(n=10, num_ds=100, m=100)


    

    




Number of datasets=100, Number of Samples=10, number of samples in testset:100
Function	 Variance		 Bias^2
g1		 0.00000000		 0.55381366
g2		 0.04447961		 0.41991505
g3		 0.11798539		 0.42180300
g4		 0.04600876		 0.09363664
g5		 0.12355812		 0.09612371
g6		 1.98816964		 0.09745515






    

In [161]:

    

report_bias_variance(n=100, num_ds=100, m=100)


    

    




Number of datasets=100, Number of Samples=100, number of samples in testset:100
Function	 Variance		 Bias^2
g1		 0.00000000		 0.58901086
g2		 0.00464569		 0.45072357
g3		 0.01270188		 0.45209526
g4		 0.00269852		 0.10926987
g5		 0.00366258		 0.10922298
g6		 0.00511599		 0.10889857






    

In [168]:

    

def fit_line_with_bias(X, Y, lambd):
    # (X'X + \lambda I)^{-1} X'Y
    res = np.matmul(X.transpose(), X)
    bias = np.multiply(np.eye(res.shape[0]), lambd)
    res = res + bias
    res = np.linalg.pinv(res)
    res = np.matmul(res, X.transpose())
    res = np.matmul(res, np.array(Y))
    return res

class RidgeRegression(LinearRegression):
    def __init__(self, X, Y, lambd):
        self.lambd = lambd
        super(RidgeRegression, self).__init__(X, Y)

    def _find_W(self, X, Y):
        return fit_line_with_bias(X, Y, self.lambd)

def report_lambd_effect(n, num_ds):
    Xs = generate_datasets(-1, 1, n, num_ds)
    Ys = generate_ys(Xs)
    m = 100 # samples for test data

    test_set = generate_datasets(-1, 1, m, 1)[0]
    test_ys = generate_ys([test_set])[0]
    lambds = [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0]
    j = 2 # degree of polynomial
    print("Lambda\t Variance\t Bias")
    for lambd in lambds:
        models = [RidgeRegression(phi(Xs[i], j), Ys[i], lambd) for i in range(num_ds)]
        phi_test_set = phi(test_set, j)
        preds = np.zeros(shape=(num_ds, m))
        for row in range(num_ds):
            pred_y = models[row].predict(phi_test_set)
            for col in range(m):
                preds[row,col] = pred_y[col]

        avgx = preds.mean(axis=0) # averaging over datasets, axis=0 is data_sets
        spread = np.zeros(shape=(num_ds, m))
        for col in range(m):
            for row in range(num_ds):
                spread[row, col] = abs(preds[row,col] - avgx[col])
        variance = np.sum(spread ** 2) / (float(num_ds * m))

        bias_2 = 0.
        for col in range(m):
            bias_2 += (avgx[col] - test_ys[col])**2
        bias_2 = bias_2 / (m)
        print("%g\t %g\t %g" % (lambd, variance, bias_2))

report_lambd_effect(100, 100)


    

    




Lambda	 Variance	 Bias
0.001	 0.00291082	 0.110499
0.003	 0.00290983	 0.110496
0.01	 0.00290636	 0.110483
0.03	 0.00289664	 0.110451
0.1	 0.00286453	 0.110368
0.3	 0.00278789	 0.110376
1	 0.00265098	 0.11267

In [194]:

    

from scipy import io
test_file = '../data/phishing-test.mat'
train_file = '../data/phishing-train.mat'

def transform_features(X):
    newX = np.zeros(shape=(X.shape[0], 0))
    for i in range(X.shape[1]):
        vals = set(X[:, i])
        if len(vals) == 3: # break into three boolean cols
            for val in vals:
                new_col = map(lambda a: int(a == val), X[:, i])
                newX = np.insert(newX, newX.shape[1], new_col, axis=1)
        elif -1 in vals:
            new_col = map(lambda a: int(a > 0), X[:, i]) # replace -1 with 0
            newX = np.insert(newX, newX.shape[1], new_col, axis=1)
        else:
            newX = np.insert(newX, newX.shape[1], X[:, i], axis=1)
    return newX

def load_data(fn):
    mat = scipy.io.loadmat(train_file)
    Y = mat['label']
    X = mat['features']
    return transform_features(X), Y.flatten()

def store_svm_vectors(X, Y, file_name):
    assert(len(X) == len(Y))
    with open(file_name, 'wb') as f:
        for i in range(len(Y)):
            f.write("%d " % Y[i]) # label
            pairs = map(lambda t: "%d:%d"% t, enumerate(X[i]))
            f.write(" ".join(pairs)) # attributes
            f.write("\n")

X, Y = load_data(train_file)
testX, testY = load_data(test_file)

store_svm_vectors(X, Y, 'train.data')
store_svm_vectors(testX, testY, 'test.data')


    

In [228]:

    

%%script bash
pow(){
    python -c "from sys import argv as a; print(pow(int(a[1]), int(a[2])))" $1 $2
}

millis(){  python -c "import time; print(int(time.time()*1000))"; }

linear_kernel(){
    echo "Linear Kernel"
    START=$(millis)
    counter=0
    for i in $(seq -6 2); do
        c=$(pow 4 $i)
        echo "C=$c"
        svm-train -t 0 -c $c -v 3 train.data | grep "Accuracy"
        ((counter++))
    done;
    END=$(millis)
    DIFF=$(echo "$END - $START" | bc)
    echo "Total time taken:"$DIFF ", $counter experiments"
    echo "Average taken :" $(echo "$DIFF / $counter" | bc) "ms"
}

poly_kernel(){
    echo "Polynomial Kernel"
    START=$(millis)
    counter=0
    for i in $(seq -3 7); do
        c=$(pow 4 $i)
        for deg in $(seq 1 3); do
            echo "C=$c, degree=$deg"
            svm-train -t 1 -c $c -d $deg -v 3 train.data | grep "Accuracy"
            ((counter++))
        done;
    done;
    END=$(millis)
    DIFF=$(echo "$END - $START" | bc)
    echo "Total time taken:"$DIFF ", $counter experiments"
    echo "Average taken :" $(echo "$DIFF / $counter" | bc) "ms"
}
RBF_kernel(){
    echo "RBF Kernel"
    START=$(millis)
    counter=0
    for i in $(seq -3 7); do
        c=$(pow 4 $i)
        for j in $(seq -7 -1); do
            gamma=$(pow 4 $j)
            echo "C=$c, gamma=$gamma"
            svm-train -t 2 -c $c -g $gamma -v 3 train.data | grep "Accuracy"
            ((counter++))
        done;
    done;
    END=$(millis)
    DIFF=$(echo "$END - $START" | bc)
    echo "Total time taken:"$DIFF ", $counter experiments"
    echo "Average taken :" $(echo "$DIFF / $counter" | bc) "ms"
}

linear_kernel
poly_kernel
RBF_kernel


    

    




Linear Kernel
C=0.000244140625
Cross Validation Accuracy = 55.75%
C=0.0009765625
Cross Validation Accuracy = 88.2%
C=0.00390625
Cross Validation Accuracy = 91.35%
C=0.015625
Cross Validation Accuracy = 92.75%
C=0.0625
Cross Validation Accuracy = 93.9%
C=0.25
Cross Validation Accuracy = 94.6%
C=1
Cross Validation Accuracy = 94.65%
C=4
Cross Validation Accuracy = 94.5%
C=16
Cross Validation Accuracy = 94.4%
Total time taken:2823 , 9 experiments
Average taken : 313 ms
Polynomial Kernel
C=0.015625, degree=1
Cross Validation Accuracy = 55.75%
C=0.015625, degree=2
Cross Validation Accuracy = 55.75%
C=0.015625, degree=3
Cross Validation Accuracy = 55.75%
C=0.0625, degree=1
Cross Validation Accuracy = 90.35%
C=0.0625, degree=2
Cross Validation Accuracy = 89%
C=0.0625, degree=3
Cross Validation Accuracy = 74.9%
C=0.25, degree=1
Cross Validation Accuracy = 91.15%
C=0.25, degree=2
Cross Validation Accuracy = 92.15%
C=0.25, degree=3
Cross Validation Accuracy = 92.05%
C=1, degree=1
Cross Validation Accuracy = 92.85%
C=1, degree=2
Cross Validation Accuracy = 93.3%
C=1, degree=3
Cross Validation Accuracy = 92.8%
C=4, degree=1
Cross Validation Accuracy = 94.35%
C=4, degree=2
Cross Validation Accuracy = 94.9%
C=4, degree=3
Cross Validation Accuracy = 95.1%
C=16, degree=1
Cross Validation Accuracy = 94.3%
C=16, degree=2
Cross Validation Accuracy = 95.95%
C=16, degree=3
Cross Validation Accuracy = 96.4%
C=64, degree=1
Cross Validation Accuracy = 94.15%
C=64, degree=2
Cross Validation Accuracy = 96.65%
C=64, degree=3
Cross Validation Accuracy = 97%
C=256, degree=1
Cross Validation Accuracy = 94.5%
C=256, degree=2
Cross Validation Accuracy = 96.8%
C=256, degree=3
Cross Validation Accuracy = 96.9%
C=1024, degree=1
Cross Validation Accuracy = 94.5%
C=1024, degree=2
Cross Validation Accuracy = 96.5%
C=1024, degree=3
Cross Validation Accuracy = 96.55%
C=4096, degree=1
Cross Validation Accuracy = 94.5%
C=4096, degree=2
Cross Validation Accuracy = 96.75%
C=4096, degree=3
Cross Validation Accuracy = 96.6%
C=16384, degree=1
Cross Validation Accuracy = 94.7%
C=16384, degree=2
Cross Validation Accuracy = 96.45%
C=16384, degree=3
Cross Validation Accuracy = 96.6%
Total time taken:14874 , 33 experiments
Average taken : 450 ms
RBF Kernel
C=0.015625, gamma=6.103515625e-05
Cross Validation Accuracy = 55.75%
C=0.015625, gamma=0.000244140625
Cross Validation Accuracy = 55.75%
C=0.015625, gamma=0.0009765625
Cross Validation Accuracy = 55.75%
C=0.015625, gamma=0.00390625
Cross Validation Accuracy = 55.75%
C=0.015625, gamma=0.015625
Cross Validation Accuracy = 56%
C=0.015625, gamma=0.0625
Cross Validation Accuracy = 87.3%
C=0.015625, gamma=0.25
Cross Validation Accuracy = 60.3%
C=0.0625, gamma=6.103515625e-05
Cross Validation Accuracy = 55.75%
C=0.0625, gamma=0.000244140625
Cross Validation Accuracy = 55.75%
C=0.0625, gamma=0.0009765625
Cross Validation Accuracy = 55.75%
C=0.0625, gamma=0.00390625
Cross Validation Accuracy = 64.4%
C=0.0625, gamma=0.015625
Cross Validation Accuracy = 90.55%
C=0.0625, gamma=0.0625
Cross Validation Accuracy = 92.2%
C=0.0625, gamma=0.25
Cross Validation Accuracy = 92.05%
C=0.25, gamma=6.103515625e-05
Cross Validation Accuracy = 55.75%
C=0.25, gamma=0.000244140625
Cross Validation Accuracy = 55.75%
C=0.25, gamma=0.0009765625
Cross Validation Accuracy = 66.6%
C=0.25, gamma=0.00390625
Cross Validation Accuracy = 90.85%
C=0.25, gamma=0.015625
Cross Validation Accuracy = 91.4%
C=0.25, gamma=0.0625
Cross Validation Accuracy = 93.1%
C=0.25, gamma=0.25
Cross Validation Accuracy = 95.55%
C=1, gamma=6.103515625e-05
Cross Validation Accuracy = 55.75%
C=1, gamma=0.000244140625
Cross Validation Accuracy = 67.85%
C=1, gamma=0.0009765625
Cross Validation Accuracy = 91.05%
C=1, gamma=0.00390625
Cross Validation Accuracy = 91.4%
C=1, gamma=0.015625
Cross Validation Accuracy = 93.5%
C=1, gamma=0.0625
Cross Validation Accuracy = 95.9%
C=1, gamma=0.25
Cross Validation Accuracy = 97.15%
C=4, gamma=6.103515625e-05
Cross Validation Accuracy = 67.95%
C=4, gamma=0.000244140625
Cross Validation Accuracy = 90.85%
C=4, gamma=0.0009765625
Cross Validation Accuracy = 91.45%
C=4, gamma=0.00390625
Cross Validation Accuracy = 93.35%
C=4, gamma=0.015625
Cross Validation Accuracy = 94.9%
C=4, gamma=0.0625
Cross Validation Accuracy = 96.8%
C=4, gamma=0.25
Cross Validation Accuracy = 96.95%
C=16, gamma=6.103515625e-05
Cross Validation Accuracy = 90.85%
C=16, gamma=0.000244140625
Cross Validation Accuracy = 91.5%
C=16, gamma=0.0009765625
Cross Validation Accuracy = 93.45%
C=16, gamma=0.00390625
Cross Validation Accuracy = 94.55%
C=16, gamma=0.015625
Cross Validation Accuracy = 95.85%
C=16, gamma=0.0625
Cross Validation Accuracy = 97.1%
C=16, gamma=0.25
Cross Validation Accuracy = 96.7%
C=64, gamma=6.103515625e-05
Cross Validation Accuracy = 91.5%
C=64, gamma=0.000244140625
Cross Validation Accuracy = 93.45%
C=64, gamma=0.0009765625
Cross Validation Accuracy = 94.45%
C=64, gamma=0.00390625
Cross Validation Accuracy = 94.5%
C=64, gamma=0.015625
Cross Validation Accuracy = 96.75%
C=64, gamma=0.0625
Cross Validation Accuracy = 97%
C=64, gamma=0.25
Cross Validation Accuracy = 96.7%
C=256, gamma=6.103515625e-05
Cross Validation Accuracy = 93.45%
C=256, gamma=0.000244140625
Cross Validation Accuracy = 94.4%
C=256, gamma=0.0009765625
Cross Validation Accuracy = 94.3%
C=256, gamma=0.00390625
Cross Validation Accuracy = 95.9%
C=256, gamma=0.015625
Cross Validation Accuracy = 97.2%
C=256, gamma=0.0625
Cross Validation Accuracy = 96.85%
C=256, gamma=0.25
Cross Validation Accuracy = 96.7%
C=1024, gamma=6.103515625e-05
Cross Validation Accuracy = 94.4%
C=1024, gamma=0.000244140625
Cross Validation Accuracy = 94.4%
C=1024, gamma=0.0009765625
Cross Validation Accuracy = 94.6%
C=1024, gamma=0.00390625
Cross Validation Accuracy = 96.7%
C=1024, gamma=0.015625
Cross Validation Accuracy = 96.9%
C=1024, gamma=0.0625
Cross Validation Accuracy = 96.85%
C=1024, gamma=0.25
Cross Validation Accuracy = 96.7%
C=4096, gamma=6.103515625e-05
Cross Validation Accuracy = 94.4%
C=4096, gamma=0.000244140625
Cross Validation Accuracy = 94.35%
C=4096, gamma=0.0009765625
Cross Validation Accuracy = 96.2%
C=4096, gamma=0.00390625
Cross Validation Accuracy = 97.1%
C=4096, gamma=0.015625
Cross Validation Accuracy = 96.95%
C=4096, gamma=0.0625
Cross Validation Accuracy = 96.85%
C=4096, gamma=0.25
Cross Validation Accuracy = 96.7%
C=16384, gamma=6.103515625e-05
Cross Validation Accuracy = 94.25%
C=16384, gamma=0.000244140625
Cross Validation Accuracy = 94.7%
C=16384, gamma=0.0009765625
Cross Validation Accuracy = 96.55%
C=16384, gamma=0.00390625
Cross Validation Accuracy = 96.75%
C=16384, gamma=0.015625
Cross Validation Accuracy = 96.95%
C=16384, gamma=0.0625
Cross Validation Accuracy = 96.85%
C=16384, gamma=0.25
Cross Validation Accuracy = 96.7%
Total time taken:26188 , 77 experiments
Average taken : 340 ms

In [128]:

    

import subprocess
from time import time
import re

LINEAR = 0; POLY = 1; RBF = 2 #kernels
train_data = "train.data"
test_data = "test.data"

def svm_train(kernel, data, verbose=False, model_file='', **args):
    arg_str = " ".join(map(lambda p: '-%s %s' % (str(p[0]), str(p[1])), args.items()))
    cmd = "svm-train -t %d %s %s %s" % (kernel, arg_str, data, model_file)
    if verbose:
        print(cmd)
    t = time()
    res = subprocess.check_output(cmd.split()).split('\n')
    if 'v' in args:
        res = filter(lambda line: 'Accuracy' in line, res)
        return float(re.search("=\s*(\d*(\.\d*)?)%$", res[0].strip()).group(1)), ((time() - t)/ args['v'])
    return None

def svm_predict(data, model, output):
    cmd = "svm-predict %s %s %s" % (data, model, output)
    return subprocess.check_output(cmd.split()).strip()


    

In [130]:

    

best = float('-inf')
best_params = None

print("LINEAR KERNEL\n#\t\tC\tAvgTime\t\tAccuracy%")
Cs = map(lambda x: pow(4, x), range(-6, 3))
for i,C in enumerate(Cs):
    res, t = svm_train(POLY, train_data, v=3, c=C)
    print("%d\t %11g\t%f\t%.2f"%(i+1, C, t, res))
    if res > best:
        best = res
        best_params = "Linear Kernel, C=%f" % C

print("POLYNOMIAL KERNEL\n#\t\t C\tDegree\tAvgTime\t\tAccuracy%")
Cs = map(lambda x: pow(4, x), range(-3, 8))
degs = [1,2,3]
ctr = 0
for C in Cs:
    for deg in degs:
        ctr += 1
        res, t = svm_train(POLY, train_data, v=3, c=C, d=deg)
        print("%d\t %11g\t%d\t%f\t%.2f"%(ctr, C, deg, t, res))
        if res > best:
            best = res
            best_params = "Plynomial Kernel, C=%f, degree=%d" % (C, deg)


print("RBF KERNEL\n#\t\t C\tGamma\t\tAvgTime\t\tAccuracy%")
Cs = map(lambda x: pow(4, x), range(-3, 8))
gammas = map(lambda x: pow(4, x), range(-7, 0))
ctr = 0
for C in Cs:
    for gamma in gammas:
        ctr += 1
        res, t = svm_train(RBF, train_data, v=3, c=C, g=gamma)
        print("%d\t %11g\t%f\t%f\t%.2f"%(ctr, C, gamma, t, res))
        if res > best:
            best = res
            best_params = "RBF Kernel, C=%f, gamma=%f" % (C, gamma)

print("## BEST RESULTS")
print(best_params)
print("Best Accuracy: %.2f" % best)


    

    




LINEAR KERNEL
#		C	AvgTime		Accuracy%
1	 0.000244141	0.213693	55.75
2	 0.000976562	0.201526	55.75
3	  0.00390625	0.197195	55.75
4	    0.015625	0.190556	55.75
5	      0.0625	0.196112	74.90
6	        0.25	0.147726	92.05
7	           1	0.100852	92.80
8	           4	0.065552	95.10
9	          16	0.053958	96.40
POLYNOMIAL KERNEL
#		 C	Degree	AvgTime		Accuracy%
1	    0.015625	1	0.193540	55.75
2	    0.015625	2	0.201199	55.75
3	    0.015625	3	0.200625	55.75
4	      0.0625	1	0.174386	90.35
5	      0.0625	2	0.198155	89.00
6	      0.0625	3	0.198131	74.90
7	        0.25	1	0.103184	91.15
8	        0.25	2	0.116746	92.15
9	        0.25	3	0.147319	92.05
10	           1	1	0.073813	92.85
11	           1	2	0.079659	93.30
12	           1	3	0.100888	92.80
13	           4	1	0.056519	94.35
14	           4	2	0.063589	94.90
15	           4	3	0.067277	95.10
16	          16	1	0.054508	94.30
17	          16	2	0.057410	95.95
18	          16	3	0.055024	96.40
19	          64	1	0.056362	94.15
20	          64	2	0.050879	96.65
21	          64	3	0.051658	97.00
22	         256	1	0.068880	94.50
23	         256	2	0.053696	96.80
24	         256	3	0.049736	96.90
25	        1024	1	0.154141	94.50
26	        1024	2	0.056765	96.50
27	        1024	3	0.053674	96.55
28	        4096	1	0.381076	94.50
29	        4096	2	0.059383	96.75
30	        4096	3	0.056649	96.60
31	       16384	1	1.563308	94.70
32	       16384	2	0.065256	96.45
33	       16384	3	0.055341	96.60
RBF KERNEL
#		 C	Gamma		AvgTime		Accuracy%
1	    0.015625	0.000061	0.196857	55.75
2	    0.015625	0.000244	0.199696	55.75
3	    0.015625	0.000977	0.193203	55.75
4	    0.015625	0.003906	0.201737	55.75
5	    0.015625	0.015625	0.200773	56.00
6	    0.015625	0.062500	0.198204	87.30
7	    0.015625	0.250000	0.200537	60.30
8	      0.0625	0.000061	0.199040	55.75
9	      0.0625	0.000244	0.200937	55.75
10	      0.0625	0.000977	0.196819	55.75
11	      0.0625	0.003906	0.203154	64.40
12	      0.0625	0.015625	0.165145	90.55
13	      0.0625	0.062500	0.128067	92.20
14	      0.0625	0.250000	0.172642	92.05
15	        0.25	0.000061	0.200721	55.75
16	        0.25	0.000244	0.199568	55.75
17	        0.25	0.000977	0.199016	66.60
18	        0.25	0.003906	0.154790	90.85
19	        0.25	0.015625	0.103935	91.40
20	        0.25	0.062500	0.082446	93.10
21	        0.25	0.250000	0.115713	95.55
22	           1	0.000061	0.196611	55.75
23	           1	0.000244	0.198686	67.85
24	           1	0.000977	0.152969	91.05
25	           1	0.003906	0.099703	91.40
26	           1	0.015625	0.074173	93.50
27	           1	0.062500	0.063382	95.90
28	           1	0.250000	0.085199	97.15
29	           4	0.000061	0.197674	67.95
30	           4	0.000244	0.151894	90.85
31	           4	0.000977	0.102306	91.45
32	           4	0.003906	0.072589	93.35
33	           4	0.015625	0.056027	94.90
34	           4	0.062500	0.054397	96.80
35	           4	0.250000	0.074961	96.95
36	          16	0.000061	0.154246	90.85
37	          16	0.000244	0.097384	91.50
38	          16	0.000977	0.074993	93.45
39	          16	0.003906	0.057111	94.55
40	          16	0.015625	0.053216	95.85
41	          16	0.062500	0.052186	97.10
42	          16	0.250000	0.075502	96.70
43	          64	0.000061	0.097574	91.50
44	          64	0.000244	0.069576	93.45
45	          64	0.000977	0.058329	94.45
46	          64	0.003906	0.056718	94.50
47	          64	0.015625	0.054045	96.75
48	          64	0.062500	0.049619	97.00
49	          64	0.250000	0.074851	96.70
50	         256	0.000061	0.071727	93.45
51	         256	0.000244	0.058747	94.40
52	         256	0.000977	0.056307	94.30
53	         256	0.003906	0.055413	95.90
54	         256	0.015625	0.054942	97.20
55	         256	0.062500	0.047539	96.85
56	         256	0.250000	0.074552	96.70
57	        1024	0.000061	0.057464	94.40
58	        1024	0.000244	0.057587	94.40
59	        1024	0.000977	0.056671	94.60
60	        1024	0.003906	0.066289	96.70
61	        1024	0.015625	0.057694	96.90
62	        1024	0.062500	0.049985	96.85
63	        1024	0.250000	0.076784	96.70
64	        4096	0.000061	0.055695	94.40
65	        4096	0.000244	0.058861	94.35
66	        4096	0.000977	0.073948	96.20
67	        4096	0.003906	0.086435	97.10
68	        4096	0.015625	0.065810	96.95
69	        4096	0.062500	0.050349	96.85
70	        4096	0.250000	0.074639	96.70
71	       16384	0.000061	0.060005	94.25
72	       16384	0.000244	0.080497	94.70
73	       16384	0.000977	0.114950	96.55
74	       16384	0.003906	0.110726	96.75
75	       16384	0.015625	0.065220	96.95
76	       16384	0.062500	0.048876	96.85
77	       16384	0.250000	0.073846	96.70
## BEST RESULTS
RBF Kernel, C=256.000000, gamma=0.015625
Best Accuracy: 97.20

In [129]:

    

from sklearn.svm import SVC

#The highest was RBF 
#Cross Validation Accuracy = 97.2%
C=256; gamma=0.015625

model = "best_model.dat" 
svm_train(RBF, train_data, model_file=model, c=C, g=gamma)
print("Test Accuracy::")
print(svm_predict(test_data, model, "/dev/null"))


    

    




Test Accuracy::
Accuracy = 98.5% (1970/2000) (classification)

In [ ]: