Upload data to floydhub check http://docs.floydhub.com/commands/data/


In [ ]:
# floyd run --mode jupyter --gpu --env theano:py2 --data <path>:data

In [ ]:
# if bcolz gives error, uncomment and run
# !pip install bcolz
# if keras gives error on importing l2, uncomment and run following
# !pip uninstall -y keras
# !pip install keras==1.2.2

In [ ]:


In [25]:
import os, sys
current_dir = os.getcwd()
LESSON_HOME_DIR = current_dir
DATA_HOME_DIR = current_dir+'/data/dogscatsredux/'

In [ ]:
sys.path.insert(1, os.path.join(sys.path[0], '..'))

#import modules
from utils import *
from vgg16 import Vgg16

#Instantiate plotting tool
#In Jupyter notebooks, you will need to run this command before doing any plotting
%matplotlib inline

In [4]:
DATA_HOME_DIR


Out[4]:
'/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/'

In [8]:
#Create directories
%cd $DATA_HOME_DIR
%mkdir valid
%mkdir results
%mkdir -p sample/train
%mkdir -p sample/test
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown
!ls -ltr


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux
mkdir: valid: File exists
mkdir: results: File exists
total 1668976
-rw-r--r--  1 nikhil  staff  284478493 Aug 26 13:23 test.zip
-rw-r--r--  1 nikhil  staff  569918665 Aug 26 13:31 train.zip
-rw-r--r--  1 nikhil  staff     113903 Aug 26 13:31 sample_submission.csv
drwxr-xr-x  2 nikhil  staff         68 Aug 27 20:00 results
drwxr-xr-x  3 nikhil  staff        102 Aug 27 20:00 test
drwxr-xr-x  2 nikhil  staff         68 Aug 27 20:02 valid
drwxr-xr-x  6 nikhil  staff        204 Aug 27 20:02 sample

In [11]:
from __future__ import division,print_function

import os, json
from glob import glob
import numpy as np
np.set_printoptions(precision=4, linewidth=100)
from matplotlib import pyplot as plt

In [12]:
%cd $DATA_HOME_DIR/train
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(2000): 
    os.rename(shuf[i], DATA_HOME_DIR+'/valid/' + shuf[i])


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/train
/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/train

In [13]:
from shutil import copyfile
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(200): 
    copyfile(shuf[i], DATA_HOME_DIR+'/sample/train/' + shuf[i])

In [14]:
%cd $DATA_HOME_DIR/valid
g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(50): 
    copyfile(shuf[i], DATA_HOME_DIR+'/sample/valid/' + shuf[i])


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/valid

In [ ]:
from shutil import copyfile
DATA_HOME_DIR
g = glob('*.jpg')
shuf = np.random.permutation(g)

for i in range(200): 
    copyfile(shuf[i], DATA_HOME_DIR+'/sample/test/' + shuf[i])

In [15]:
#Divide cat/dog images into separate directories

%cd $DATA_HOME_DIR/sample/train
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/sample/train

In [16]:
%cd $DATA_HOME_DIR/sample/valid
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/

%cd $DATA_HOME_DIR/valid
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/

%cd $DATA_HOME_DIR/train
%mkdir cats
%mkdir dogs
%mv cat.*.jpg cats/
%mv dog.*.jpg dogs/


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/sample/valid
/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/valid
/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/train

In [17]:
# Create single 'unknown' class for test set
%cd $DATA_HOME_DIR/test
%mv *.jpg unknown/


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/test

In [37]:
%cd $DATA_HOME_DIR

#Set path to sample/ path if desired
path = DATA_HOME_DIR #'/sample/'
path = DATA_HOME_DIR + 'sample/'

test_path = path + 'test/'
results_path=DATA_HOME_DIR + 'results/'
train_path=path + 'train/'
valid_path=path + 'valid/'


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux

In [29]:
vgg = Vgg16()

In [63]:
%%time
#Finetune the model
batch_size=64
batches = vgg.get_batches(train_path, batch_size=batch_size)
val_batches = vgg.get_batches(valid_path, batch_size=batch_size*2)
vgg.finetune(batches)
vgg.fit(batches, val_batches, nb_epoch=1)
#Not sure if we set this for all fits
vgg.model.optimizer.lr = 0.01
latest_weights_filename = 'ft0d.h5'
vgg.model.save_weights(results_path+latest_weights_filename)


Found 200 images belonging to 2 classes.
Found 50 images belonging to 2 classes.
Epoch 1/1
200/200 [==============================] - 171s - loss: 1.2301 - acc: 0.6100 - val_loss: 0.0914 - val_acc: 0.9800
CPU times: user 5min 16s, sys: 17.2 s, total: 5min 33s
Wall time: 2min 57s

In [70]:



Out[70]:
'/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux/sample/test/'

In [71]:
batches, preds = vgg.test(test_path, batch_size = batch_size*2)


Found 200 images belonging to 1 classes.

In [79]:
filenames = batches.filenames
filenames[:5]


Out[79]:
array([[  2.3505e-03,   9.9765e-01],
       [  9.9577e-01,   4.2272e-03],
       [  9.2571e-01,   7.4287e-02],
       [  9.8730e-01,   1.2697e-02],
       [  9.9992e-01,   7.6639e-05]], dtype=float32)

In [80]:
preds[:5]


Out[80]:
array([[  2.3505e-03,   9.9765e-01],
       [  9.9577e-01,   4.2272e-03],
       [  9.2571e-01,   7.4287e-02],
       [  9.8730e-01,   1.2697e-02],
       [  9.9992e-01,   7.6639e-05]], dtype=float32)

In [81]:
#You can verify the column ordering by viewing some images
from PIL import Image
Image.open(test_path + filenames[2])


Out[81]:

In [82]:
save_array(results_path + 'test_preds.dat', preds)
save_array(results_path + 'filenames.dat', filenames)

In [90]:
isDog = preds[:,1]
isDog[:5]


Out[90]:
array([  9.9765e-01,   4.2272e-03,   7.4287e-02,   1.2697e-02,   7.6639e-05], dtype=float32)

In [85]:
ids = np.array([int(f[8:f.find('.')]) for f in filenames])

In [89]:
ids[:5]


Out[89]:
array([10022, 10040, 10081, 10228, 10286])

In [88]:
subm = np.stack([ids,isDog], axis=1)
subm[:5]


Out[88]:
array([[  1.0022e+04,   9.9765e-01],
       [  1.0040e+04,   4.2272e-03],
       [  1.0081e+04,   7.4287e-02],
       [  1.0228e+04,   1.2697e-02],
       [  1.0286e+04,   7.6639e-05]])

In [91]:
%cd $DATA_HOME_DIR
submission_file_name = 'submission1.csv'
np.savetxt(submission_file_name, subm, fmt='%d,%.5f', header='id,label', comments='')


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs/data/dogscatsredux

In [93]:
from IPython.display import FileLink
%cd $LESSON_HOME_DIR
FileLink('data/dogscatsredux/'+submission_file_name)


/Users/nikhil/workspace/www/ml/fastai/courses/deeplearning1/nbs

In [ ]: