In [2]:

    

%matplotlib inline

import matplotlib
import autograd.numpy as np
import matplotlib.pyplot as plt
import random
import math
from autograd import grad

def generateChevronData():
    xBounds = [-50, 50]
    yBounds = [-50, 50]
    totalPoints = 100
    
    points = []
    targets = []
    
    for i in range(0, totalPoints):
        x = random.randint(xBounds[0], xBounds[1])
        y = random.randint(yBounds[0], yBounds[1])
        
        if x >= y and x <= -y:
            points.append([1, x/50.0,y/50.0])
            targets.append(0)
        else:
            points.append([1, x/50.0,y/50.0])
            targets.append(1)
        
    return np.array(points), np.array(targets)

def plotLine(points):
    xs = [x[1] for x in points]
    ys = [y[2] for y in points]
    
    plt.plot(xs, ys, color='r',linestyle='-')

def plotScatter(points):
    xs = [x[1] for x in points]
    ys = [y[2] for y in points]
    
    plt.scatter(xs, ys)


    

In [3]:

    

def sigmoid(phi):
    return 1.0/(1.0 + np.exp(-phi))

def MSE(weights):
    localPoints = np.array(list(map(lambda x: (x[1] - weights[1])**2 + (x[2] - weights[2])**2 <= radius**2, points)))
    predictions = logisticPrediction(weights, points)
    
    s = 0
    for i in range(0, len(points)):
        if localPoints[i]:
            s += (targets[i] - predictions[i])**2
    
    return 1.0/2.0 * s
#     return 1.0/2.0 * np.sum(np.power((targets - predictions), 2))

# def localLogLoss(weights):
#     predictions = logisticPrediction(weights, points)
#     print(targets)
#     print(predictions)
#     probs = predictions * targets + (1 - predictions) * (1 - targets)
#     print(predictions)
#     print(targets)
#     print(np.log(probs))
#     return -(1/len(points)) * np.sum(np.log(probs))

# def localLogLoss(weights, example):
#     return (predict(weights, example))

def logisticPrediction(weights, p):
    ins = np.array(list(map(lambda x: predict(weights, x), p)))
    return ins
    
def predict(weights, i):
    return sigmoid((i[2] - weights[2]) - weights[0] * (i[1] - weights[1]))


    

In [4]:

    

def computeGradient(weights, example, target):
    prediction = predict(weights, example)
    E = -np.log(target * prediction + (1-target) * (1-prediction))
    dE_dO = computeErrorDifferential(prediction, target)
    
    dO_dZ = prediction * (1-prediction)
    
    dZ_dy = -1
    dZ_dm = weights[1] - example[1]
    dZ_dx = weights[0]
    
    dE_dZ = dE_dO * dO_dZ
    
    grad = np.zeros(3)#[0.0, 0.0, 0.0]
    grad[0] = dZ_dm * dE_dZ
    grad[1] = dZ_dx * dE_dZ
    grad[2] = dZ_dy * dE_dZ
    
    return grad, E

def computeErrorDifferential(prediction, target):
    return -(target - prediction)
#     print(prediction, target)
#     if target == 1:
#         return -1/np.log(prediction)
    
#     return 1/np.log(1-prediction)


    

In [5]:

    

def trainBoundaryHunter(weights):
    iPoints = []
#     weights = np.array([0.0, 0.5, -1.0])
#     trainingGradient = grad(MSE)
    
    print("Initial Loss: ", MSE(weights))
    for i in range(0, 10000):
#         g = trainingGradient(weights) * 0.01
        if i % 1000 == 0:
            print()
            iPoints.append(np.copy(weights))
            print("Loss Before: " + str(MSE(weights)))

        weights, error = computeStep(weights)
#         weights -= g
    
        if i % 1000 == 0:
            print("Loss After [i = " + str(i) + "]: " + str(MSE(weights)))
            print(weights)
            
    print("Trained Loss: ", MSE(weights))    
    print("Weights: ", weights)
    return weights, iPoints

def computeStep(weights):
    totalG = np.zeros(3)
    totalE = 0
    
    localPoints = np.array(list(map(lambda x: (x[1] - weights[1])**2 + (x[2] - weights[2])**2 <= radius**2, points)))

    
    for i in range(0, len(points)):
        if not localPoints[i]:
            continue
        
        g, error = computeGradient(weights, points[i], targets[i])
        totalE += error
        totalG += g     
        
    totalG = totalG * (1/len(points))
    totalE = totalE * (1/len(points))
        
    weights -= totalG * 0.01
    return weights, totalE


    

In [10]:

    

random.seed(1234)
points, targets = generateChevronData()

plt.axis([-1.5, 1.5, -1.5, 1.5])

# Plot points on graph
c1 = []
c2 = []

for i in range(0, len(points)):
    if targets[i] == 0:
        c1.append(points[i])
    else:
        c2.append(points[i])

print("Type 0: ", len(c1))
print("Type 1: ", len(c2))
        
plotScatter(c1)
plotScatter(c2)

radius = 0.5

boundaryHunters = []

for i in range(0, 3):
    print("\n\nBoundary Hunter: " + str(i) + "\n")
    weights = np.random.uniform(-0.5, 0.5, 3)
    weights, iPoints = trainBoundaryHunter(weights)
    boundaryHunters.append(weights)
    
    plotLine(iPoints)
    plt.scatter(weights[1], weights[2])
    plt.plot([-1.0, 1.0], [weights[2] + weights[0]*((-1) - weights[1]), weights[2] + weights[0]*(1 - weights[1])], 'k-')

    x = np.linspace(-1.5, 1.5, 500)
    y = np.linspace(-1.5, 1.5, 500)
    X, Y = np.meshgrid(x,y)
    F = ((X - weights[1]))**2 + ((Y - weights[2]))**2 - radius**2
    plt.contour(X,Y,F,[0])

plt.gca().set_aspect('equal')

plt.show()


    

    




Type 0:  35
Type 1:  65


Boundary Hunter: 0

Initial Loss:  1.63204891482

Loss Before: 1.63204891482
Loss After [i = 0]: 1.63177092365
[-0.39671528 -0.20921633  0.43178954]

Loss Before: 1.47661871083
Loss After [i = 1000]: 1.47637251409
[-0.38363837 -0.26778192  0.28139062]

Loss Before: 1.92949262018
Loss After [i = 2000]: 1.92917428007
[-0.36813708 -0.32515023  0.12942068]

Loss Before: 1.9262833571
Loss After [i = 3000]: 1.92609961192
[-0.33676569 -0.37688884 -0.01733495]

Loss Before: 1.62486207501
Loss After [i = 4000]: 1.62475450205
[-0.30745253 -0.41262915 -0.12790452]

Loss Before: 1.62910912867
Loss After [i = 5000]: 1.62906299569
[-0.2773988  -0.4341265  -0.20129866]

Loss Before: 1.62270669344
Loss After [i = 6000]: 1.62269168936
[-0.24650195 -0.44421684 -0.23925384]

Loss Before: 1.69421514509
Loss After [i = 7000]: 1.69420127119
[-0.21159601 -0.44737463 -0.25282676]

Loss Before: 1.68046762413
Loss After [i = 8000]: 1.68045399952
[-0.1750584  -0.44857226 -0.25901199]

Loss Before: 1.66695923437
Loss After [i = 9000]: 1.66694583962
[-0.13874726 -0.44945462 -0.26462529]
Trained Loss:  1.65367297772
Weights:  [-0.10269841 -0.45007203 -0.26972778]


Boundary Hunter: 1

Initial Loss:  2.53941709941

Loss Before: 2.53941709941
Loss After [i = 0]: 2.53930519247
[ 0.22902056  0.48837608 -0.21189279]

Loss Before: 2.3962357145
Loss After [i = 1000]: 2.39618832264
[ 0.1932579   0.50465662 -0.2884653 ]

Loss Before: 2.15242586398
Loss After [i = 2000]: 2.15240624956
[ 0.15690714  0.51163133 -0.32777545]

Loss Before: 2.13372580379
Loss After [i = 3000]: 2.13370793959
[ 0.12094239  0.51494136 -0.35153468]

Loss Before: 2.2640451613
Loss After [i = 4000]: 2.26402931765
[ 0.08265031  0.51717256 -0.37287914]

Loss Before: 2.24857605712
Loss After [i = 5000]: 2.2485609394
[ 0.04567609  0.51803571 -0.38624751]

Loss Before: 2.14177306104
Loss After [i = 6000]: 2.14176280054
[ 0.0127634   0.51815759 -0.38958202]

Loss Before: 2.13159684872
Loss After [i = 7000]: 2.13158675675
[-0.01910208  0.51815283 -0.39106303]

Loss Before: 2.121588498
Loss After [i = 8000]: 2.1215785732
[-0.0507022   0.51810047 -0.39256172]

Loss Before: 2.11174661245
Loss After [i = 9000]: 2.1117368534
[-0.08203709  0.51800012 -0.39407322]
Trained Loss:  2.10206972538
Weights:  [-0.11307607  0.51785201 -0.3955912 ]


Boundary Hunter: 2

Initial Loss:  1.57251838808

Loss Before: 1.57251838808
Loss After [i = 0]: 1.57228737387
[ 0.09411633 -0.23900396  0.37296391]

Loss Before: 1.43732007456
Loss After [i = 1000]: 1.4371451118
[ 0.10586394 -0.2248203   0.23072084]

Loss Before: 1.5409057692
Loss After [i = 2000]: 1.54083446495
[ 0.1163416  -0.21135172  0.10994913]

Loss Before: 1.7108418076
Loss After [i = 3000]: 1.71077549335
[ 0.126206   -0.202234    0.03485385]

Loss Before: 1.85277985139
Loss After [i = 4000]: 1.85269953418
[ 0.14238671 -0.18872528 -0.06602894]

Loss Before: 1.71696968012
Loss After [i = 5000]: 1.7169387375
[ 0.16249258 -0.17930927 -0.12824186]

Loss Before: 1.77547179346
Loss After [i = 6000]: 1.77546476127
[ 0.18001334 -0.17422591 -0.15816985]

Loss Before: 1.84232467058
Loss After [i = 7000]: 1.84232091716
[ 0.1963891  -0.17201562 -0.16995927]

Loss Before: 1.8387156717
Loss After [i = 8000]: 1.8387121979
[ 0.21267197 -0.17005785 -0.17953955]

Loss Before: 1.83535988277
Loss After [i = 9000]: 1.8353566377
[ 0.22884149 -0.16820485 -0.18794014]
Trained Loss:  1.83221192337
Weights:  [ 0.2448754  -0.16646609 -0.19528651]






    

In [ ]:

    




    

In [ ]:

    




    

In [ ]: