In [1]:
%matplotlib inline
#%matplotlib

In [110]:
import matplotlib.pyplot as plt
from matplotlib import image as mpimg
from scipy import sqrt, pi, arctan2, cos, sin, ndimage, fftpack, stats
from skimage import exposure, measure, feature
from pandas import read_csv, DataFrame
from pandas import concat as pd_concat
from PIL import Image
import cStringIO
import urllib2
from numpy.random import rand
from numpy import ones, zeros, concatenate

from sklearn.ensemble import ExtraTreesClassifier
from sklearn import svm

In [46]:
feat_df = pd.read_csv("tmp/HogFeatures.csv")
full_df = pd.read_csv("machine_learn/HOG/csv_features/hog_features_9_NewTraining_Faces_everyones.csv")

In [142]:
def save_ImageOfHog(grey_img,ImageOfHog,rgb_img):
    f,ax=plt.subplots(1,3,figsize=(20,20))
    ax[0].imshow(rgb_img)
    ax[1].imshow(grey_img,cmap='gray')
    ax[2].imshow(ImageOfHog,cmap='gray',interpolation='nearest')
    titles=["RGB Image","Greyscale input for HOG","Image of Histogram Oriented Gradients (HOG)"]
    for cnt,axis in enumerate(ax):
        axis.set_title(titles[cnt],fontdict={'fontsize':20.})
        [label.set_visible(False) for label in axis.get_xticklabels()]
        [label.set_visible(False) for label in axis.get_yticklabels()]
    plt.savefig("tmp/image_of_hog.png",dpi=200)

def CalcHog_FeaturesAndImage_ForOneImage(grey_img,image_name,rgb_img):
    feat = zeros((1,900)) #People_All_9.csv Food_All_9.csv
    #get hog features
    blocks = feature.hog(grey_img, orientations=9, pixels_per_cell=(100,100), cells_per_block=(5,5), visualise=False, normalise=True) #People_All_9.csv Food_All_9.csv
    #slightly diff params for better hog visualization
    junk_block,ImageOfHog=feature.hog(grey_img, pixels_per_cell=(10,10), cells_per_block=(30,30),visualise=True,normalise=True)
    
    if(len(blocks) == 900): #People_All_9.csv Food_All_9.csv
        feat[0] = blocks

    name_df=DataFrame()
    name_df["image_name"]= image_name
    feat_df= DataFrame(feat)
    final_df=pd_concat([name_df,feat_df],axis=1) 
    final_df.to_csv("tmp/HogFeatures.csv")
    
    save_ImageOfHog(grey_img,ImageOfHog,rgb_img)

def Hog_predict_UploadImage(grey_img,image_name,rgb_img):
    CalcHog_FeaturesAndImage_ForOneImage(grey_img,image_name,rgb_img)
    feat_df= read_csv("tmp/HogFeatures.csv")
    feat_vals= feat_df.ix[:,2:].values

    root= "machine_learn/HOG/csv_features/"
    Food_df = read_csv(root+"hog_features_9_NewTraining_Food_everyones.csv")
    People_df = read_csv(root+"hog_features_9_NewTraining_Faces_everyones.csv")
    
    cTrainF = rand(len(Food_df)) > .5
    cTestF = ~cTrainF
    cTrainP = rand(len(People_df)) > .5
    cTestP = ~cTrainP
    
    TrainX_df = pd_concat([People_df[cTrainP], Food_df[cTrainF]],axis=0)
    TrainX= TrainX_df.ix[:,2:].values
    TrainY = concatenate([ones(len(People_df[cTrainP])), zeros(len(Food_df[cTrainF]))])

    ET_classifier = ExtraTreesClassifier(n_estimators=50, max_depth=None, min_samples_split=1, random_state=0)
    ET_classifier.fit(TrainX,TrainY)
    ET_prediction = ET_classifier.predict(feat_vals) 

    LinSVC_classifier = svm.LinearSVC()
    LinSVC_classifier.fit(TrainX,TrainY)
    LinSVC_prediction = LinSVC_classifier.predict(feat_vals)
    return ET_prediction, LinSVC_prediction

In [363]:
root="machine_learn/blob_hog_predict_common_url_set/"
fin=open(root+'FacesAndLimbs_shuffled_blob_features.pickle',"r")
feat_df= pickle.load(fin)
fin.close()
fin=open(root+'FacesAndLimbs_shuffled_url_answer.pickle',"r")
url_df= pickle.load(fin)
fin.close()

In [399]:
root="machine_learn/blob_hog_predict_common_url_set/"
f_faces=root+"FacesAndLimbs_shuffled_hog_features.pickle"
fin=open(f_faces)
faces_df= pickle.load(fin)
faces_df.shape


Out[399]:
(477, 902)

In [443]:
root="machine_learn/blob_hog_predict_common_url_set/"
f_hog=root+"FacesAndLimbs_shuffled_hog_predict.pickle"
f_blob=root+"FacesAndLimbs_shuffled_blob_predict.pickle"
fin=open(f_hog,"r")
hog_df= pickle.load(fin)
fin.close()
fin=open(f_blob,"r")
blob_df= pickle.load(fin)
fin.close()

In [444]:
a=set(blob_df.url.values.flatten())
b=set(hog_df.url.values.flatten())
c=a.intersection(b)
common_urls=np.array(list(c))

In [459]:
hog_df.head()


Out[459]:
url answer ET_predict LinSVC_predict
0 http://scontent-b.cdninstagram.com/hphotos-xpa... 1 1 1
1 http://scontent-b.cdninstagram.com/hphotos-xfp... 0 1 0
2 http://scontent-a.cdninstagram.com/hphotos-xpa... 1 1 1
3 http://scontent-b.cdninstagram.com/hphotos-xfp... 1 1 0
4 http://scontent-a.cdninstagram.com/hphotos-xpa... 0 1 1

5 rows × 4 columns


In [462]:
fin=open(root+'NoLims_shuffled_blob_stats.pickle')
blob_stats=pickle.load(fin)
fin.close()

In [466]:
from random import randrange
print randrange(10)


9

In [442]:
np.array(list(c))[0]


Out[442]:
'http://scontent-b.cdninstagram.com/hphotos-xpa1/t51.2885-15/1389646_659244750792796_1580007320_n.jpg'

In [394]:
len(c),url_df.URL.values.size,hog_df.URL.values.size


Out[394]:
(477, 485, 477)

In [48]:
#testing
# import matplotlib.image as mpimg
# file='uploads/engage_1ps.jpg'
# img= mpimg.imread(file)

print img.shape
plt.imshow(img)


(640, 640)
Out[48]:
<matplotlib.image.AxesImage at 0x10e341110>

In [ ]:


In [48]:


In [146]:
# file="machine_learn/training_image_urls/NewTraining_Food_everyones.txt"
# urls=np.loadtxt(file,dtype="str")
# url=urls[11]
# read= urllib2.urlopen(url).read()
# obj = Image.open( cStringIO.StringIO(read) )
# rgb_img= np.array(obj)
# # grey_img = np.array(obj.convert('L'))
# grey_img = np.array(.convert('L'))
file= 'uploads/engage_1ps.jpg'
# import matplotlib.image as mpimg
# image= mpimg.imread(file)
obj = Image.open( file )
rgb_img= np.array(obj)
grey_img = np.array(obj.convert('L'))
(et,svc)=Hog_predict_UploadImage(grey_img,file,rgb_img)



In [150]:
def interpret_int_predict(int_predict):
    food_code=0
    people_code=1
    if int_predict == food_code: return "Contains: Food"
    else: return "Contains: People"

ans=interpret_int_predict(et[0].astype('int'))
print ans


Contains: Food

In [153]:
a=1.2344423
s= "%.1f" % a
print s


1.2

make common url list to train on for Blob and Hog


In [197]:
from numpy.random import rand
from numpy import ones, zeros, concatenate
import numpy as np
from pandas import read_csv, DataFrame, concat

from sklearn.ensemble import RandomForestClassifier

In [293]:
class FacesAndLimbsTrainingSet():
    def __init__(self,Food_df,Faces_df,SkinNoFaces_df):
        self.Food= Food_df.ix[:251,:].copy()
        self.People= self.Food.copy()
        for i in np.arange(0,134):
            self.People.ix[i,:]= Faces_df.ix[i,:].copy()
        cnt=0
        for i in np.arange(134,250):
            self.People.ix[i,:]= SkinNoFaces_df.ix[cnt,:].copy()
            cnt+=1

class NoLimbsTrainingSet():
    def __init__(self,Food_df,Faces_df):
        self.Food= Food_df.ix[:251,:].copy()
        self.People= Faces_df.ix[:251,:].copy()
        
class NoFacesTrainingSet():
    def __init__(self,Food_df,SkinNoFaces_df):
        self.Food= Food_df.ix[:117,:].copy()
        self.People= SkinNoFaces_df.ix[:117,:].copy()

class Team_or_Kay_Features():
    def __init__(self,Food_df,Faces_df,SkinNoFaces_df):
        self.FacesAndLimbs= FacesAndLimbsTrainingSet(Food_df,Faces_df,SkinNoFaces_df)
        self.NoLims= NoLimbsTrainingSet(Food_df,Faces_df)
        self.NoFaces= NoFacesTrainingSet(Food_df,SkinNoFaces_df)

class AddTeamCols():
    def __init__(self, Food_KayF,Faces_KayF,SkinNoFaces_KayF, 
                Food_TeamF,Faces_TeamF,SkinNoFaces_TeamF):
        self.Food= Food_KayF.copy()
        cols= Food_TeamF.columns[2:12]
        for col in cols:
            self.Food[col]= Food_TeamF[col]
        
        self.Faces= Faces_KayF.copy()
        cols= Faces_TeamF.columns[2:12]
        for col in cols:
            self.Faces[col]= Faces_TeamF[col]

        self.SkinNoFaces= SkinNoFaces_KayF.copy()
        cols= SkinNoFaces_TeamF.columns[2:12]
        for col in cols:
            self.SkinNoFaces[col]= SkinNoFaces_TeamF[col]

Food_KayF = read_csv('machine_learn/Blob/csv_features/NewTraining_Food_everyones_KFeat_Toddmap.csv')
Faces_KayF = read_csv('machine_learn/Blob/csv_features/NewTraining_Faces_everyones_KFeat_Toddmap.csv')
SkinNoFaces_KayF = read_csv('machine_learn/Blob/csv_features/NewTraining_SkinNoFaces_everyones_KFeat_Toddmap.csv')
Food_TeamF = read_csv('machine_learn/Blob/csv_features/NewTraining_Food_everyones_TeamFeat_Toddmap.csv')
Faces_TeamF = read_csv('machine_learn/Blob/csv_features/NewTraining_Faces_everyones_TeamFeat_Toddmap.csv')
SkinNoFaces_TeamF = read_csv('machine_learn/Blob/csv_features/NewTraining_SkinNoFaces_everyones_TeamFeat_Toddmap.csv')

#team feature numbers for different definitions of Food,People
team= Team_or_Kay_Features(Food_TeamF,Faces_TeamF,SkinNoFaces_TeamF)
#kay feature numbers for different definitions of Food,People
kay= Team_or_Kay_Features(Food_KayF,Faces_KayF,SkinNoFaces_KayF)
#kay feature numbers + team feature number for skin maps for different definitions of Food,People
extend= AddTeamCols(Food_KayF,Faces_KayF,SkinNoFaces_KayF, 
            Food_TeamF,Faces_TeamF,SkinNoFaces_TeamF)
kay_extend= Team_or_Kay_Features(extend.Food,extend.Faces,extend.SkinNoFaces)

##
#make training and test sets
Food_all = kay_extend.NoLims.Food
People_all= kay_extend.NoLims.People

In [294]:
kay_extend.FacesAndLimbs.Food.shape,kay_extend.FacesAndLimbs.People.shape


Out[294]:
((250, 22), (250, 22))

In [205]:
pd.DataFrame?

In [296]:
ans_people=pd.DataFrame(np.ones(250),columns=["answer"])
ans_food= pd.DataFrame(np.zeros(250),columns=["answer"])
FacesAndLimbs_food= pd.concat([ans_food,kay_extend.FacesAndLimbs.Food.ix[:,1:]],axis=1)
FacesAndLimbs_people= pd.concat([ans_people,kay_extend.FacesAndLimbs.People.ix[:,1:]],axis=1)
FacesAndLimbs_food.shape, FacesAndLimbs_people.shape


Out[296]:
((250, 22), (250, 22))

In [298]:
FacesAndLimbs= pd.concat([FacesAndLimbs_food,FacesAndLimbs_people],axis=0)
inds=np.arange(0,FacesAndLimbs.shape[0])
FacesAndLimbs.index= inds
FacesAndLimbs.shape


Out[298]:
(500, 22)

In [299]:
def shuffle_DataFrame_rows(df):
    from random import shuffle
    inds=np.arange(0,df.shape[0])
    shf_inds= inds.copy()
    shuffle(shf_inds)
    shf_df= df.ix[shf_inds,:]
    shf_df.index= inds
    return shf_df
shf_FacesAndLimbs= shuffle_DataFrame_rows(FacesAndLimbs)

In [300]:
shf_FacesAndLimbs_url_answer= shf_FacesAndLimbs.ix[:,0:2]
shf_FacesAndLimbs_blob_features= shf_FacesAndLimbs.ix[:,2:]

In [304]:
name="FacesAndLimbs_shuffled_url_answer.pickle"
fout = open(name, 'w') 
pickle.dump(shf_FacesAndLimbs_url_answer, fout)
fout.close()

In [176]:
import pickle

In [190]:
fin=open('machine_learn/blob_hog_predict_common_url_set/FacesAndLimbs_Food_urls.pickle',"r")
test=pickle.load(fin)
fin.close()

In [308]:
fin=open('machine_learn/blob_hog_predict_common_url_set/FacesAndLimbs_shuffled_url_answer.pickle',"r")
df= pickle.load(fin)
fin.close()

In [310]:
df.ix[[0,1,3],:].head()


Out[310]:
answer URL
0 1 http://scontent-a.cdninstagram.com/hphotos-xfp...
1 1 http://scontent-b.cdninstagram.com/hphotos-xpa...
3 0 http://scontent-a.cdninstagram.com/hphotos-xfa...

3 rows × 2 columns


In [400]:
root="machine_learn/blob_hog_predict_common_url_set/"
fin=open(root+'FacesAndLimbs_shuffled_hog_features.pickle',"r")
df= pickle.load(fin)
fin.close()

In [406]:
fin=open(root+'NoLims_shuffled_url_answer.pickle',"r")
ans_url_df= pickle.load(fin)
fin.close()
ans_url_df.head()


Out[406]:
answer URL
0 0 http://scontent-a.cdninstagram.com/hphotos-xap...
1 0 http://scontent-a.cdninstagram.com/hphotos-xaf...
2 1 http://scontent-a.cdninstagram.com/hphotos-xpf...
3 0 http://scontent-a.cdninstagram.com/hphotos-xap...
4 0 http://scontent-b.cdninstagram.com/hphotos-xpa...

5 rows × 2 columns


In [356]:
root= "machine_learn/blob_hog_predict_common_url_set/"
f_url= root+"NoLims_shuffled_url_answer.pickle"
f_feat= root+"NoLims_shuffled_blob_features.pickle"
fin=open(f_url,"r")
url_df= pickle.load(fin)
fin.close()
fin=open(f_feat,"r")
feat_df= pickle.load(fin)
fin.close()

In [407]:
fin=open(root+'FacesAndLimbs_shuffled_hog_features.pickle',"r")
feat_df= pickle.load(fin)
fin.close()

In [419]:
feat_df.head()


Out[419]:
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
0 0.001531 0.000618 0.000859 0.000857 0.001740 0.000854 0.000983 0.000977 0.000938 0.001959 0.003002 0.005073 0.002494 0.001627 0.000387 0.000521 0.000369 0.000679 0.001545 0.000512 ...
1 0.000616 0.000488 0.001051 0.001497 0.004507 0.001003 0.000522 0.000275 0.000187 0.000578 0.000525 0.000945 0.001501 0.004115 0.000963 0.000457 0.000272 0.000161 0.000543 0.000420 ...
2 0.001778 0.001817 0.003051 0.003972 0.004109 0.002219 0.001398 0.001025 0.000752 0.001514 0.001395 0.002465 0.003437 0.005548 0.002723 0.001669 0.001004 0.000579 0.001689 0.001558 ...
3 0.001133 0.000804 0.001913 0.003546 0.002497 0.000735 0.000632 0.000288 0.000182 0.001470 0.000798 0.001651 0.002707 0.002421 0.000711 0.000751 0.000367 0.000203 0.001684 0.000946 ...
4 0.002722 0.002892 0.002852 0.002360 0.001647 0.001149 0.001024 0.001018 0.001028 0.002472 0.002667 0.003886 0.003310 0.002938 0.001443 0.001079 0.001264 0.001367 0.002721 0.003308 ...

5 rows × 900 columns


In [421]:
feat_df.values[0:300,2:]


Out[421]:
array([[0.001530846861716368, 0.000617743166953909, 0.0008591580481259986,
        ..., 0.002975706234539831, 0.0037044411922793705,
        0.007389815977344566],
       [0.0006162118347754087, 0.0004881422450507937,
        0.0010508977044078021, ..., 0.0026810566911931277,
        0.0024755808906880127, 0.0032021377787464556],
       [0.0017779366476333841, 0.0018169499885910005,
        0.0030506997568160196, ..., 0.0033878017874200586,
        0.002342746391686112, 0.0018609353889282338],
       ..., 
       [0.0034042497734254012, 0.0026635495231744012, 0.00405050847585656,
        ..., 0.0011329387745271037, 0.0006177702667135287,
        0.000540562647637771],
       [0.0027392567627443367, 0.0013664917657758167, 0.002436229009666602,
        ..., 0.0012435977846749534, 0.0004829243659028938,
        0.00016142796003037793],
       [0.0012490278822733583, 0.0005664109566086223, 0.001064547336096439,
        ..., 0.0006990049595140919, 0.00036526301860218625,
        0.0002587228617948082]], dtype=object)

In [351]:
url_df_2= url_df.dropna()
url_df_2.shape,url_df.shape


Out[351]:
((485, 2), (493, 2))

In [342]:
bad="http://scontent-b.cdninstagram.com/hphotos-xpa1/t51.2885-15/925501_1420836204853800_1357191103_n.jpg"
ind_gd= np.where(url_df.URL.values != bad)[0]
url_gd_df= url_df.ix[ind_gd,:]
feat_gd_df= feat_df.ix[ind_gd,:]

In [343]:
np.where(url_df.URL.values == bad)[0]


Out[343]:
array([], dtype=int64)

In [340]:
fout = open(f_feat, 'w') 
pickle.dump(feat_gd_df, fout)
fout.close()

update data frames with only URLS that EXIST


In [312]:
def get_indices_urls_that_exist(urls): 
    inds_exist=[]
    cnt=-1
    for url in df.URL.values:
        cnt+=1
        print "cnt= %d" % cnt
        try:
            read= urllib2.urlopen(url).read()
            inds_exist.append(cnt)
        except urllib2.URLError:
            continue
    return np.array(inds_exist)

root= "machine_learn/blob_hog_predict_common_url_set/"
url_1= root+"NoLims_shuffled_url_answer.pickle"
fin=open(url_1,"r")
url_df= pickle.load(fin)
fin.close()
url_1_inds= get_indices_urls_that_exist(url_df.URL.values)


cnt= 0
cnt= 1
cnt= 2
cnt= 3
cnt= 4
cnt= 5
cnt= 6
cnt= 7
cnt= 8
cnt= 9
cnt= 10
cnt= 11
cnt= 12
cnt= 13
cnt= 14
cnt= 15
cnt= 16
cnt= 17
cnt= 18
cnt= 19
cnt= 20
cnt= 21
cnt= 22
cnt= 23
cnt= 24
cnt= 25
cnt= 26
cnt= 27
cnt= 28
cnt= 29
cnt= 30
cnt= 31
cnt= 32
cnt= 33
cnt= 34
cnt= 35
cnt= 36
cnt= 37
cnt= 38
cnt= 39
cnt= 40
cnt= 41
cnt= 42
cnt= 43
cnt= 44
cnt= 45
cnt= 46
cnt= 47
cnt= 48
cnt= 49
cnt= 50
cnt= 51
cnt= 52
cnt= 53
cnt= 54
cnt= 55
cnt= 56
cnt= 57
cnt= 58
cnt= 59
cnt= 60
cnt= 61
cnt= 62
cnt= 63
cnt= 64
cnt= 65
cnt= 66
cnt= 67
cnt= 68
cnt= 69
cnt= 70
cnt= 71
cnt= 72
cnt= 73
cnt= 74
cnt= 75
cnt= 76
cnt= 77
cnt= 78
cnt= 79
cnt= 80
cnt= 81
cnt= 82
cnt= 83
cnt= 84
cnt= 85
cnt= 86
cnt= 87
cnt= 88
cnt= 89
cnt= 90
cnt= 91
cnt= 92
cnt= 93
cnt= 94
cnt= 95
cnt= 96
cnt= 97
cnt= 98
cnt= 99
cnt= 100
cnt= 101
cnt= 102
cnt= 103
cnt= 104
cnt= 105
cnt= 106
cnt= 107
cnt= 108
cnt= 109
cnt= 110
cnt= 111
cnt= 112
cnt= 113
cnt= 114
cnt= 115
cnt= 116
cnt= 117
cnt= 118
cnt= 119
cnt= 120
cnt= 121
cnt= 122
cnt= 123
cnt= 124
cnt= 125
cnt= 126
cnt= 127
cnt= 128
cnt= 129
cnt= 130
cnt= 131
cnt= 132
cnt= 133
cnt= 134
cnt= 135
cnt= 136
cnt= 137
cnt= 138
cnt= 139
cnt= 140
cnt= 141
cnt= 142
cnt= 143
cnt= 144
cnt= 145
cnt= 146
cnt= 147
cnt= 148
cnt= 149
cnt= 150
cnt= 151
cnt= 152
cnt= 153
cnt= 154
cnt= 155
cnt= 156
cnt= 157
cnt= 158
cnt= 159
cnt= 160
cnt= 161
cnt= 162
cnt= 163
cnt= 164
cnt= 165
cnt= 166
cnt= 167
cnt= 168
cnt= 169
cnt= 170
cnt= 171
cnt= 172
cnt= 173
cnt= 174
cnt= 175
cnt= 176
cnt= 177
cnt= 178
cnt= 179
cnt= 180
cnt= 181
cnt= 182
cnt= 183
cnt= 184
cnt= 185
cnt= 186
cnt= 187
cnt= 188
cnt= 189
cnt= 190
cnt= 191
cnt= 192
cnt= 193
cnt= 194
cnt= 195
cnt= 196
cnt= 197
cnt= 198
cnt= 199
cnt= 200
cnt= 201
cnt= 202
cnt= 203
cnt= 204
cnt= 205
cnt= 206
cnt= 207
cnt= 208
cnt= 209
cnt= 210
cnt= 211
cnt= 212
cnt= 213
cnt= 214
cnt= 215
cnt= 216
cnt= 217
cnt= 218
cnt= 219
cnt= 220
cnt= 221
cnt= 222
cnt= 223
cnt= 224
cnt= 225
cnt= 226
cnt= 227
cnt= 228
cnt= 229
cnt= 230
cnt= 231
cnt= 232
cnt= 233
cnt= 234
cnt= 235
cnt= 236
cnt= 237
cnt= 238
cnt= 239
cnt= 240
cnt= 241
cnt= 242
cnt= 243
cnt= 244
cnt= 245
cnt= 246
cnt= 247
cnt= 248
cnt= 249
cnt= 250
cnt= 251
cnt= 252
cnt= 253
cnt= 254
cnt= 255
cnt= 256
cnt= 257
cnt= 258
cnt= 259
cnt= 260
cnt= 261
cnt= 262
cnt= 263
cnt= 264
cnt= 265
cnt= 266
cnt= 267
cnt= 268
cnt= 269
cnt= 270
cnt= 271
cnt= 272
cnt= 273
cnt= 274
cnt= 275
cnt= 276
cnt= 277
cnt= 278
cnt= 279
cnt= 280
cnt= 281
cnt= 282
cnt= 283
cnt= 284
cnt= 285
cnt= 286
cnt= 287
cnt= 288
cnt= 289
cnt= 290
cnt= 291
cnt= 292
cnt= 293
cnt= 294
cnt= 295
cnt= 296
cnt= 297
cnt= 298
cnt= 299
cnt= 300
cnt= 301
cnt= 302
cnt= 303
cnt= 304
cnt= 305
cnt= 306
cnt= 307
cnt= 308
cnt= 309
cnt= 310
cnt= 311
cnt= 312
cnt= 313
cnt= 314
cnt= 315
cnt= 316
cnt= 317
cnt= 318
cnt= 319
cnt= 320
cnt= 321
cnt= 322
cnt= 323
cnt= 324
cnt= 325
cnt= 326
cnt= 327
cnt= 328
cnt= 329
cnt= 330
cnt= 331
cnt= 332
cnt= 333
cnt= 334
cnt= 335
cnt= 336
cnt= 337
cnt= 338
cnt= 339
cnt= 340
cnt= 341
cnt= 342
cnt= 343
cnt= 344
cnt= 345
cnt= 346
cnt= 347
cnt= 348
cnt= 349
cnt= 350
cnt= 351
cnt= 352
cnt= 353
cnt= 354
cnt= 355
cnt= 356
cnt= 357
cnt= 358
cnt= 359
cnt= 360
cnt= 361
cnt= 362
cnt= 363
cnt= 364
cnt= 365
cnt= 366
cnt= 367
cnt= 368
cnt= 369
cnt= 370
cnt= 371
cnt= 372
cnt= 373
cnt= 374
cnt= 375
cnt= 376
cnt= 377
cnt= 378
cnt= 379
cnt= 380
cnt= 381
cnt= 382
cnt= 383
cnt= 384
cnt= 385
cnt= 386
cnt= 387
cnt= 388
cnt= 389
cnt= 390
cnt= 391
cnt= 392
cnt= 393
cnt= 394
cnt= 395
cnt= 396
cnt= 397
cnt= 398
cnt= 399
cnt= 400
cnt= 401
cnt= 402
cnt= 403
cnt= 404
cnt= 405
cnt= 406
cnt= 407
cnt= 408
cnt= 409
cnt= 410
cnt= 411
cnt= 412
cnt= 413
cnt= 414
cnt= 415
cnt= 416
cnt= 417
cnt= 418
cnt= 419
cnt= 420
cnt= 421
cnt= 422
cnt= 423
cnt= 424
cnt= 425
cnt= 426
cnt= 427
cnt= 428
cnt= 429
cnt= 430
cnt= 431
cnt= 432
cnt= 433
cnt= 434
cnt= 435
cnt= 436
cnt= 437
cnt= 438
cnt= 439
cnt= 440
cnt= 441
cnt= 442
cnt= 443
cnt= 444
cnt= 445
cnt= 446
cnt= 447
cnt= 448
cnt= 449
cnt= 450
cnt= 451
cnt= 452
cnt= 453
cnt= 454
cnt= 455
cnt= 456
cnt= 457
cnt= 458
cnt= 459
cnt= 460
cnt= 461
cnt= 462
cnt= 463
cnt= 464
cnt= 465
cnt= 466
cnt= 467
cnt= 468
cnt= 469
cnt= 470
cnt= 471
cnt= 472
cnt= 473
cnt= 474
cnt= 475
cnt= 476
cnt= 477
cnt= 478
cnt= 479
cnt= 480
cnt= 481
cnt= 482
cnt= 483
cnt= 484
cnt= 485
cnt= 486
cnt= 487
cnt= 488
cnt= 489
cnt= 490
cnt= 491
cnt= 492
cnt= 493
cnt= 494
cnt= 495
cnt= 496
cnt= 497
cnt= 498
cnt= 499
cnt= 500
cnt= 501

In [320]:
path=root+"NoLims_shuffled_blob_features.pickle"
fin=open(path,"r")
blob_feat_df= pickle.load(fin)
fin.close()

df_exist= df.ix[url_1_inds,:]
blob_feat_df_exist= blob_feat_df.ix[url_1_inds,:]
df_exist.shape,df.shape,blob_feat_df_exist.shape,blob_feat_df.shape


Out[320]:
((494, 2), (502, 2), (494, 20), (502, 20))

In [322]:
fout = open("NoLims_shuffled_url_answer.pickle", 'w') 
pickle.dump(df_exist, fout)
fout.close()

In [323]:
url_2= root+"FacesAndLimbs_shuffled_url_answer.pickle"
fin=open(url_2,"r")
url_df= pickle.load(fin)
fin.close()
url_2_inds= get_indices_urls_that_exist(url_df.URL.values)


cnt= 0
cnt= 1
cnt= 2
cnt= 3
cnt= 4
cnt= 5
cnt= 6
cnt= 7
cnt= 8
cnt= 9
cnt= 10
cnt= 11
cnt= 12
cnt= 13
cnt= 14
cnt= 15
cnt= 16
cnt= 17
cnt= 18
cnt= 19
cnt= 20
cnt= 21
cnt= 22
cnt= 23
cnt= 24
cnt= 25
cnt= 26
cnt= 27
cnt= 28
cnt= 29
cnt= 30
cnt= 31
cnt= 32
cnt= 33
cnt= 34
cnt= 35
cnt= 36
cnt= 37
cnt= 38
cnt= 39
cnt= 40
cnt= 41
cnt= 42
cnt= 43
cnt= 44
cnt= 45
cnt= 46
cnt= 47
cnt= 48
cnt= 49
cnt= 50
cnt= 51
cnt= 52
cnt= 53
cnt= 54
cnt= 55
cnt= 56
cnt= 57
cnt= 58
cnt= 59
cnt= 60
cnt= 61
cnt= 62
cnt= 63
cnt= 64
cnt= 65
cnt= 66
cnt= 67
cnt= 68
cnt= 69
cnt= 70
cnt= 71
cnt= 72
cnt= 73
cnt= 74
cnt= 75
cnt= 76
cnt= 77
cnt= 78
cnt= 79
cnt= 80
cnt= 81
cnt= 82
cnt= 83
cnt= 84
cnt= 85
cnt= 86
cnt= 87
cnt= 88
cnt= 89
cnt= 90
cnt= 91
cnt= 92
cnt= 93
cnt= 94
cnt= 95
cnt= 96
cnt= 97
cnt= 98
cnt= 99
cnt= 100
cnt= 101
cnt= 102
cnt= 103
cnt= 104
cnt= 105
cnt= 106
cnt= 107
cnt= 108
cnt= 109
cnt= 110
cnt= 111
cnt= 112
cnt= 113
cnt= 114
cnt= 115
cnt= 116
cnt= 117
cnt= 118
cnt= 119
cnt= 120
cnt= 121
cnt= 122
cnt= 123
cnt= 124
cnt= 125
cnt= 126
cnt= 127
cnt= 128
cnt= 129
cnt= 130
cnt= 131
cnt= 132
cnt= 133
cnt= 134
cnt= 135
cnt= 136
cnt= 137
cnt= 138
cnt= 139
cnt= 140
cnt= 141
cnt= 142
cnt= 143
cnt= 144
cnt= 145
cnt= 146
cnt= 147
cnt= 148
cnt= 149
cnt= 150
cnt= 151
cnt= 152
cnt= 153
cnt= 154
cnt= 155
cnt= 156
cnt= 157
cnt= 158
cnt= 159
cnt= 160
cnt= 161
cnt= 162
cnt= 163
cnt= 164
cnt= 165
cnt= 166
cnt= 167
cnt= 168
cnt= 169
cnt= 170
cnt= 171
cnt= 172
cnt= 173
cnt= 174
cnt= 175
cnt= 176
cnt= 177
cnt= 178
cnt= 179
cnt= 180
cnt= 181
cnt= 182
cnt= 183
cnt= 184
cnt= 185
cnt= 186
cnt= 187
cnt= 188
cnt= 189
cnt= 190
cnt= 191
cnt= 192
cnt= 193
cnt= 194
cnt= 195
cnt= 196
cnt= 197
cnt= 198
cnt= 199
cnt= 200
cnt= 201
cnt= 202
cnt= 203
cnt= 204
cnt= 205
cnt= 206
cnt= 207
cnt= 208
cnt= 209
cnt= 210
cnt= 211
cnt= 212
cnt= 213
cnt= 214
cnt= 215
cnt= 216
cnt= 217
cnt= 218
cnt= 219
cnt= 220
cnt= 221
cnt= 222
cnt= 223
cnt= 224
cnt= 225
cnt= 226
cnt= 227
cnt= 228
cnt= 229
cnt= 230
cnt= 231
cnt= 232
cnt= 233
cnt= 234
cnt= 235
cnt= 236
cnt= 237
cnt= 238
cnt= 239
cnt= 240
cnt= 241
cnt= 242
cnt= 243
cnt= 244
cnt= 245
cnt= 246
cnt= 247
cnt= 248
cnt= 249
cnt= 250
cnt= 251
cnt= 252
cnt= 253
cnt= 254
cnt= 255
cnt= 256
cnt= 257
cnt= 258
cnt= 259
cnt= 260
cnt= 261
cnt= 262
cnt= 263
cnt= 264
cnt= 265
cnt= 266
cnt= 267
cnt= 268
cnt= 269
cnt= 270
cnt= 271
cnt= 272
cnt= 273
cnt= 274
cnt= 275
cnt= 276
cnt= 277
cnt= 278
cnt= 279
cnt= 280
cnt= 281
cnt= 282
cnt= 283
cnt= 284
cnt= 285
cnt= 286
cnt= 287
cnt= 288
cnt= 289
cnt= 290
cnt= 291
cnt= 292
cnt= 293
cnt= 294
cnt= 295
cnt= 296
cnt= 297
cnt= 298
cnt= 299
cnt= 300
cnt= 301
cnt= 302
cnt= 303
cnt= 304
cnt= 305
cnt= 306
cnt= 307
cnt= 308
cnt= 309
cnt= 310
cnt= 311
cnt= 312
cnt= 313
cnt= 314
cnt= 315
cnt= 316
cnt= 317
cnt= 318
cnt= 319
cnt= 320
cnt= 321
cnt= 322
cnt= 323
cnt= 324
cnt= 325
cnt= 326
cnt= 327
cnt= 328
cnt= 329
cnt= 330
cnt= 331
cnt= 332
cnt= 333
cnt= 334
cnt= 335
cnt= 336
cnt= 337
cnt= 338
cnt= 339
cnt= 340
cnt= 341
cnt= 342
cnt= 343
cnt= 344
cnt= 345
cnt= 346
cnt= 347
cnt= 348
cnt= 349
cnt= 350
cnt= 351
cnt= 352
cnt= 353
cnt= 354
cnt= 355
cnt= 356
cnt= 357
cnt= 358
cnt= 359
cnt= 360
cnt= 361
cnt= 362
cnt= 363
cnt= 364
cnt= 365
cnt= 366
cnt= 367
cnt= 368
cnt= 369
cnt= 370
cnt= 371
cnt= 372
cnt= 373
cnt= 374
cnt= 375
cnt= 376
cnt= 377
cnt= 378
cnt= 379
cnt= 380
cnt= 381
cnt= 382
cnt= 383
cnt= 384
cnt= 385
cnt= 386
cnt= 387
cnt= 388
cnt= 389
cnt= 390
cnt= 391
cnt= 392
cnt= 393
cnt= 394
cnt= 395
cnt= 396
cnt= 397
cnt= 398
cnt= 399
cnt= 400
cnt= 401
cnt= 402
cnt= 403
cnt= 404
cnt= 405
cnt= 406
cnt= 407
cnt= 408
cnt= 409
cnt= 410
cnt= 411
cnt= 412
cnt= 413
cnt= 414
cnt= 415
cnt= 416
cnt= 417
cnt= 418
cnt= 419
cnt= 420
cnt= 421
cnt= 422
cnt= 423
cnt= 424
cnt= 425
cnt= 426
cnt= 427
cnt= 428
cnt= 429
cnt= 430
cnt= 431
cnt= 432
cnt= 433
cnt= 434
cnt= 435
cnt= 436
cnt= 437
cnt= 438
cnt= 439
cnt= 440
cnt= 441
cnt= 442
cnt= 443
cnt= 444
cnt= 445
cnt= 446
cnt= 447
cnt= 448
cnt= 449
cnt= 450
cnt= 451
cnt= 452
cnt= 453
cnt= 454
cnt= 455
cnt= 456
cnt= 457
cnt= 458
cnt= 459
cnt= 460
cnt= 461
cnt= 462
cnt= 463
cnt= 464
cnt= 465
cnt= 466
cnt= 467
cnt= 468
cnt= 469
cnt= 470
cnt= 471
cnt= 472
cnt= 473
cnt= 474
cnt= 475
cnt= 476
cnt= 477
cnt= 478
cnt= 479
cnt= 480
cnt= 481
cnt= 482
cnt= 483
cnt= 484
cnt= 485
cnt= 486
cnt= 487
cnt= 488
cnt= 489
cnt= 490
cnt= 491
cnt= 492
cnt= 493
cnt= 494
cnt= 495
cnt= 496
cnt= 497
cnt= 498
cnt= 499
cnt= 500
cnt= 501

In [326]:
path=root+"FacesAndLimbs_shuffled_blob_features.pickle"
fin=open(path,"r")
blob_feat_df= pickle.load(fin)
fin.close()

url_df_exist= url_df.ix[url_2_inds,:]
blob_feat_df_exist= blob_feat_df.ix[url_2_inds,:]
url_df_exist.shape,url_df.shape,blob_feat_df_exist.shape,blob_feat_df.shape


Out[326]:
((494, 2), (500, 2), (494, 20), (500, 20))

In [328]:
fout = open("FacesAndLimbs_shuffled_url_answer.pickle", 'w') 
pickle.dump(url_df_exist, fout)
fout.close()

image of HOG


In [69]:
file="machine_learn/training_image_urls/NewTraining_Faces_everyones.txt"
urls=np.loadtxt(file,dtype="str")
url=urls[11]
read= urllib2.urlopen(url).read()
obj = Image.open( cStringIO.StringIO(read) )
img = np.array(obj.convert('L'))
print img.shape
plt.imshow(img)


(640, 640)
Out[69]:
<matplotlib.image.AxesImage at 0x110dda810>

In [102]:
block,ImageOfHog=feature.hog(img, pixels_per_cell=(10,10), cells_per_block=(30,30),visualise=True)#,normalise=True)

In [140]:
file="machine_learn/training_image_urls/NewTraining_Food_everyones.txt"
urls=np.loadtxt(file,dtype="str")
url=urls[11]
read= urllib2.urlopen(url).read()
obj = Image.open( cStringIO.StringIO(read) )
rgb_img= np.array(obj)
grey_img = np.array(obj.convert('L'))
f,ax=plt.subplots(1,3,figsize=(20,20))
# ax= axis.flatten()
ax[0].imshow(rgb_img)
# ax[1].imshow(np.zeros((10,10)),cmap='gray_r')
ax[1].imshow(grey_img,cmap='gray')
ax[2].imshow(ImageOfHog,cmap='gray',interpolation='nearest')
titles=["RGB Image","Greyscale input for HOG","Image of Histogram Oriented Gradients (HOG)"]
for cnt,axis in enumerate(ax):
    axis.set_title(titles[cnt],fontdict={'fontsize':20.})
    [label.set_visible(False) for label in axis.get_xticklabels()]
    [label.set_visible(False) for label in axis.get_yticklabels()]
plt.savefig("tmp/image_of_hog.png",dpi=200)



In [104]:
block,ImageOfHog=feature.hog(img, pixels_per_cell=(10,10), cells_per_block=(30,30),visualise=True,normalise=True)
plt.imshow(ImageOfHog)


Out[104]:
<matplotlib.image.AxesImage at 0x11f4d25d0>

In [52]:
feat_df= read_csv("tmp/HogFeatures.csv")
feat_vals= feat_df.ix[:,2:].values

In [55]:
feature.hog?

In [ ]: