Generate competition submission



In [1]:

    
INPUT_DIR = '../../input/kaggle-sea-lion/'
OUTPUT_DIR = '../../output/kaggle-sea-lion/20/'

IMAGE_DIMS = (42,42,3)
LABEL_DIMS = (6,)
LOAD_WEIGHTS_FILE = INPUT_DIR + '05/weights-medium1-42x42-0.94.h5'
LOAD_MODEL_FILE = None
DEBUG = True
IMAGE_SLICE = slice(1400,1800)
#IMAGE_SLICE = slice(0,99999)

NR_SHARDS = 4
RECREATE_OUTPUT = True



In [2]:

    
%matplotlib inline
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
import sklearn
import os
import glob
import cv2
from multiprocessing import Pool
import multiprocessing

from modules.logging import logger
import modules.utils as utils
from modules.utils import Timer
import modules.logging as logging
import modules.cnn as cnn
import modules.lions as lions
import modules.shards as shards
import modules.objectdetect as objectdetect









    



Using TensorFlow backend.

Classify and count lions on each image

Find lions on all images



In [3]:

    
#from MismatchedTrainImages.txt
MISMATCHED = [3, 7, 9, 21, 30, 34, 71, 81, 89, 97, 151, 184, 215, 234, 242, 268, 290, 311, 331, 344, 380, 384, 406, 421, 469, 475, 490, 499, 507, 530, 531, 605, 607, 614, 621, 638, 644, 687, 712, 721, 767, 779, 781, 794, 800, 811, 839, 840, 869, 882, 901, 903, 905, 909, 913, 927, 946]
FORCE_IMAGES = [42]
FORCE_IMAGES = None



In [4]:

    
def process_shard_lions(shard_group, shard_id):
    t = Timer('PROCESSING SHARD {}'.format(shard_id))

    output_dir = shard_group.shard_dir(shard_id)
    logging.setup_file_logger(output_dir + 'out.log')
    
    logger.info('Load CNN model')
    #lion simple cnn forward time: ~10ms
    print(LOAD_WEIGHTS_FILE)
    model = lions.convnet_medium1_lion_keras(IMAGE_DIMS)
    model.load_weights(LOAD_WEIGHTS_FILE)    

    #DECLARE FUNCTIONS INSIDE PROCESS INSTANCE
    """ Returns (score, label) """
    def evaluate_region_all_classes(region_img):
        y_pred = model.predict(np.array([region_img]))
        ylp = utils.onehot_to_label(np.array(y_pred))
        return y_pred[0][ylp[0]], ylp[0]

    
    def detect_lions(image, evaluate_function):
        #search for lions
        region_generator = objectdetect.sliding_window_generator(image, step=(19,19), window=IMAGE_DIMS, pyramid_max_layers=1)
        detections, imgs = objectdetect.evaluate_regions(region_generator, evaluate_function, filter_score_min=0.97, 
                                                         filter_labels=(0,1,2,3,4), apply_non_max_suppression=True, 
                                                         supression_overlap_threshold=0.1, threads=None)
        #calculate detection class distribution
        detected_lions = np.zeros(LABEL_DIMS[0]-1, dtype='int')
        for detection in detections:
            label = int(detection[5])
            detected_lions[label] += 1 

        if(DEBUG):
            def detection_to_colortext(detection):
                score = detection[4]
                text = str(int(detection[5])) + ' ' + '{0:.2f}'.format(score)
                c = int(score*255)
                return (0,0,c), text

            img2 = image.copy()
            objectdetect.draw_detections(detections, img2, detection_to_colortext)
            utils.show_image(img2, size=60, is_bgr=True)
    #        patches = objectdetect.extract_patches(detections, img)
    #        utils.show_images(patches[0:50], size=2, cols=10, is_bgr=True)   

        return detected_lions.tolist()

    if(shard_group.shard_done(shard_id)):
        logger.warning('shard {} already processed. Skipping'.format(shard_id))

    else:
        image_paths = shard_group.shard_items(shard_id)

        total_detected_lions = []

        for image_path in image_paths:
            base = os.path.basename(image_path)
            fn = os.path.splitext(base)
            if(fn[1]!='.jpg'):
                logger.info('ignoring non jpg image. filename=' + base)
                continue

            train_id = int(fn[0])

            if(FORCE_IMAGES!=None and train_id not in FORCE_IMAGES):
                continue

            t = Timer('processing photo ' + image_path)
            image_raw = cv2.imread(image_path)
            image_raw = image_raw[IMAGE_SLICE]
            detected_lions = detect_lions(image_raw, evaluate_region_all_classes)
            logger.info('image ' + str(train_id))
            logger.info('total detections: ' + str(np.sum(np.array(detected_lions))))
            logger.info('class detections: ' + str(detected_lions))
            total_detected_lions += [[train_id] + detected_lions]
            t.stop()

        logger.info('GENERATE SUBMISSION FILE')
        submission_file = output_dir + 'submission.csv'
        df = pd.DataFrame(total_detected_lions, columns=('test_id','adult_males','subadult_males','adult_females','juveniles','pups'))
        df.to_csv(submission_file, index=False)

        logger.info('detection result exported to ' + submission_file)        
        shard_group.mark_done(shard_id)

        t.stop()



In [5]:

    
logger.info('==== PROCESSING SHARDS IN PARALLEL ====')

logger.info('preparing shards')
images_dir = INPUT_DIR + "Train/"
image_paths = [images_dir+n for n in os.listdir(images_dir)]

shard_group = shards.ShardGroup(image_paths, NR_SHARDS, OUTPUT_DIR, recreate_shards_dir=RECREATE_OUTPUT)









    



2017-06-07 20:56:31,221 INFO ==== PROCESSING SHARDS IN PARALLEL ====
2017-06-07 20:56:31,222 INFO preparing shards



In [6]:

    
shard_group.start_processing(process_shard_lions, shard_ids=None, threads=-1)
logger.info('==== ALL DONE ====')









    



2017-06-07 20:56:31,254 INFO Using 4 parallel tasks
2017-06-07 20:56:32,334 INFO > [started] PROCESSING SHARD 3...
2017-06-07 20:56:32,344 INFO Load CNN model






    



../../input/kaggle-sea-lion/05/weights-medium1-42x42-0.94.h5






    



/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, activation="softmax", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-07 20:56:32,773 INFO found 3 items for shard 3
2017-06-07 20:56:32,776 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/49.jpg...






    



(400, 4992, 3)






    



2017-06-07 20:56:33,051 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 20:56:33,053 INFO > [started] sliding_window...






    



0/400 [>.........................] 0% 0s sliding window





    



2017-06-07 20:56:34,330 INFO > [started] PROCESSING SHARD 4...
2017-06-07 20:56:34,339 INFO Load CNN model






    



../../input/kaggle-sea-lion/05/weights-medium1-42x42-0.94.h5






    



/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, activation="softmax", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))






    



19/400 [=>........................] 4% 1s sliding window





    



2017-06-07 20:56:35,042 INFO found 2 items for shard 4
2017-06-07 20:56:35,053 INFO ignoring non jpg image. filename=MismatchedTrainImages.txt
2017-06-07 20:56:35,055 INFO ignoring non jpg image. filename=.ipynb_checkpoints
2017-06-07 20:56:35,062 INFO GENERATE SUBMISSION FILE
2017-06-07 20:56:35,081 INFO detection result exported to ../../output/kaggle-sea-lion/20/4/submission.csv
2017-06-07 20:56:35,095 INFO > [done]    PROCESSING SHARD 4 (764.221 ms)
2017-06-07 20:56:35,330 INFO > [started] PROCESSING SHARD 1...
2017-06-07 20:56:35,348 INFO Load CNN model






    



../../input/kaggle-sea-lion/05/weights-medium1-42x42-0.94.h5






    



/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, activation="softmax", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-07 20:56:35,988 INFO found 3 items for shard 1
2017-06-07 20:56:36,000 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/43.jpg...






    



(400, 5616, 3)






    



2017-06-07 20:56:36,643 INFO pyramid layer=0 image=(400, 5616, 3) scale=1
2017-06-07 20:56:36,650 INFO > [started] sliding_window...






    



57/400 [====>.....................] 14% 6s remaining=37s sliding window





    



2017-06-07 20:56:41,340 INFO > [started] PROCESSING SHARD 2...
2017-06-07 20:56:41,357 INFO Load CNN model






    



../../input/kaggle-sea-lion/05/weights-medium1-42x42-0.94.h5






    



/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), activation="relu", kernel_initializer="glorot_uniform", padding="same")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, activation="relu", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, activation="softmax", kernel_initializer="glorot_uniform")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))






    



38/400 [==>.......................] 9% 5s sliding window sliding window





    



2017-06-07 20:56:42,298 INFO found 5 items for shard 2
2017-06-07 20:56:42,314 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/44.jpg...






    



(400, 4992, 3)






    



2017-06-07 20:56:42,824 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 20:56:42,827 INFO > [started] sliding_window...






    



399/400 [=========================>] 99% 56s remaining=0s sliding windoww





    



2017-06-07 20:57:29,147 INFO > [done]    sliding_window (56094.160 ms)
2017-06-07 20:57:29,156 INFO > [started] non_max_suppression. boxes=0...
2017-06-07 20:57:29,165 INFO > [done]    non_max_suppression. boxes=0 (8.778 ms)






    



304/400 [===================>......] 76% 54s remaining=17s sliding window





    












    



2017-06-07 20:57:32,844 INFO image 49
2017-06-07 20:57:32,850 INFO total detections: 0
2017-06-07 20:57:32,854 INFO class detections: [0, 0, 0, 0, 0]
2017-06-07 20:57:32,865 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/49.jpg (60088.730 ms)
2017-06-07 20:57:32,876 INFO ignoring non jpg image. filename=train.csv
2017-06-07 20:57:32,917 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/45.jpg...






    



304/400 [===================>......] 76% 50s remaining=15s sliding window





    



2017-06-07 20:57:33,485 INFO > [started] non_max_suppression. boxes=1000439...






    



(400, 4992, 3)






    



2017-06-07 20:57:33,608 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 20:57:33,620 INFO > [started] sliding_window...






    



38/400 [==>.......................] 9% 5s sliding windows sliding windoww





    



2017-06-07 20:57:39,162 INFO > [done]    non_max_suppression. boxes=1000439 (5677.417 ms)






    



399/400 [=========================>] 99% 59s remaining=0s sliding windoww





    



2017-06-07 20:57:42,282 INFO > [done]    sliding_window (59455.108 ms)
2017-06-07 20:57:42,297 INFO > [started] non_max_suppression. boxes=53124...
2017-06-07 20:57:42,508 INFO > [done]    non_max_suppression. boxes=53124 (210.953 ms)






    



76/400 [=====>....................] 19% 11s remaining=49s sliding windoww





    












    



2017-06-07 20:57:46,415 INFO image 44
2017-06-07 20:57:46,424 INFO total detections: 3
2017-06-07 20:57:46,443 INFO class detections: [3, 0, 0, 0, 0]
2017-06-07 20:57:46,462 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/44.jpg (64147.645 ms)
2017-06-07 20:57:46,479 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/47.jpg...






    



(400, 5616, 3)






    



2017-06-07 20:57:47,430 INFO pyramid layer=0 image=(400, 5616, 3) scale=1
2017-06-07 20:57:47,433 INFO > [started] sliding_window...






    



399/400 [=========================>] 99% 71s remaining=0s sliding window





    



2017-06-07 20:57:47,959 INFO > [done]    sliding_window (71309.094 ms)
2017-06-07 20:57:47,962 INFO > [started] non_max_suppression. boxes=789707...






    



38/400 [==>.......................] 9% 5s sliding window3s sliding window





    



2017-06-07 20:57:53,800 INFO > [done]    non_max_suppression. boxes=789707 (5837.987 ms)






    



152/400 [==========>...............] 38% 23s remaining=37s sliding window





    












    



2017-06-07 20:57:57,123 INFO image 43
2017-06-07 20:57:57,146 INFO total detections: 63
2017-06-07 20:57:57,153 INFO class detections: [63, 0, 0, 0, 0]
2017-06-07 20:57:57,191 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/43.jpg (81191.262 ms)
2017-06-07 20:57:57,196 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/46.jpg...






    



(400, 5616, 3)






    



2017-06-07 20:57:58,310 INFO pyramid layer=0 image=(400, 5616, 3) scale=1
2017-06-07 20:57:58,341 INFO > [started] sliding_window...






    



190/400 [============>.............] 47% 35s remaining=39s sliding window





    



2017-06-07 20:58:22,903 INFO > [started] non_max_suppression. boxes=1000033...






    



228/400 [==============>...........] 56% 41s remaining=31s sliding window





    



2017-06-07 20:58:29,078 INFO > [done]    non_max_suppression. boxes=1000033 (6175.542 ms)






    



399/400 [=========================>] 99% 66s remaining=0s sliding windoww





    



2017-06-07 20:58:40,312 INFO > [done]    sliding_window (66692.441 ms)
2017-06-07 20:58:40,317 INFO > [started] non_max_suppression. boxes=952820...






    



323/400 [====================>.....] 80% 58s remaining=14s sliding window





    



2017-06-07 20:58:46,923 INFO > [done]    non_max_suppression. boxes=952820 (6606.026 ms)






    



342/400 [=====================>....] 85% 61s remaining=10s sliding window





    












    



2017-06-07 20:58:49,892 INFO image 45
2017-06-07 20:58:49,899 INFO total detections: 66
2017-06-07 20:58:49,915 INFO class detections: [66, 0, 0, 0, 0]
2017-06-07 20:58:49,947 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/45.jpg (77029.515 ms)
2017-06-07 20:58:49,969 INFO GENERATE SUBMISSION FILE
2017-06-07 20:58:50,000 INFO detection result exported to ../../output/kaggle-sea-lion/20/3/submission.csv
2017-06-07 20:58:50,009 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/45.jpg (77092.044 ms)






    



399/400 [=========================>] 99% 65s remaining=0s sliding windoww





    



2017-06-07 20:58:52,610 INFO > [done]    sliding_window (65177.456 ms)
2017-06-07 20:58:52,617 INFO > [started] non_max_suppression. boxes=363322...






    



304/400 [===================>......] 76% 55s remaining=17s sliding window





    



2017-06-07 20:58:53,652 INFO > [done]    non_max_suppression. boxes=363322 (1034.973 ms)






    












    



2017-06-07 20:58:55,521 INFO image 47
2017-06-07 20:58:55,523 INFO total detections: 14
2017-06-07 20:58:55,525 INFO class detections: [14, 0, 0, 0, 0]
2017-06-07 20:58:55,537 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/47.jpg (69058.322 ms)
2017-06-07 20:58:55,547 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/48.jpg...






    



323/400 [====================>.....] 80% 57s remaining=13s sliding window(400, 4992, 3)






    



2017-06-07 20:58:56,064 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 20:58:56,067 INFO > [started] sliding_window...






    



0/400 [>.........................] 0% 0s sliding window





    



2017-06-07 20:58:56,555 INFO > [started] non_max_suppression. boxes=1000274...






    



38/400 [==>.......................] 9% 4s sliding window





    



2017-06-07 20:59:00,932 INFO > [done]    non_max_suppression. boxes=1000274 (4377.724 ms)






    



399/400 [=========================>] 99% 67s remaining=0s sliding windoww





    



2017-06-07 20:59:06,452 INFO > [done]    sliding_window (68110.694 ms)
2017-06-07 20:59:06,468 INFO > [started] non_max_suppression. boxes=624258...






    



95/400 [======>...................] 23% 11s remaining=38s sliding window





    



2017-06-07 20:59:09,263 INFO > [done]    non_max_suppression. boxes=624258 (2794.417 ms)






    



114/400 [=======>..................] 28% 13s remaining=34s sliding window





    












    



2017-06-07 20:59:11,537 INFO image 46
2017-06-07 20:59:11,544 INFO total detections: 49
2017-06-07 20:59:11,547 INFO class detections: [49, 0, 0, 0, 0]
2017-06-07 20:59:11,549 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/46.jpg (74353.588 ms)
2017-06-07 20:59:11,553 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/41.jpg...






    



133/400 [========>.................] 33% 15s remaining=31s sliding window(400, 4992, 3)






    



2017-06-07 20:59:12,039 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 20:59:12,042 INFO > [started] sliding_window...






    



76/400 [=====>....................] 19% 10s remaining=44s sliding windoww





    



2017-06-07 20:59:24,829 INFO > [started] non_max_suppression. boxes=1000037...






    



133/400 [========>.................] 33% 16s remaining=33s sliding window





    



2017-06-07 20:59:29,457 INFO > [done]    non_max_suppression. boxes=1000037 (4628.340 ms)






    



247/400 [===============>..........] 61% 31s remaining=19s sliding window





    



2017-06-07 20:59:44,352 INFO > [started] non_max_suppression. boxes=1000573...






    



285/400 [==================>.......] 71% 36s remaining=14s sliding window





    



2017-06-07 20:59:49,630 INFO > [done]    non_max_suppression. boxes=1000573 (5277.520 ms)






    



399/400 [=========================>] 99% 56s remaining=0s sliding windoww





    



2017-06-07 20:59:52,990 INFO > [done]    sliding_window (56922.929 ms)
2017-06-07 20:59:52,996 INFO > [started] non_max_suppression. boxes=828390...






    



399/400 [=========================>] 99% 44s remaining=0s sliding window





    



2017-06-07 20:59:56,964 INFO > [done]    sliding_window (44921.839 ms)
2017-06-07 20:59:56,966 INFO > [started] non_max_suppression. boxes=221691...
2017-06-07 20:59:57,517 INFO > [done]    non_max_suppression. boxes=828390 (4521.442 ms)
2017-06-07 20:59:57,604 INFO > [done]    non_max_suppression. boxes=221691 (638.096 ms)






    












    



2017-06-07 20:59:59,954 INFO image 41
2017-06-07 20:59:59,956 INFO total detections: 10
2017-06-07 20:59:59,957 INFO class detections: [10, 0, 0, 0, 0]
2017-06-07 20:59:59,959 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/41.jpg (48406.384 ms)
2017-06-07 20:59:59,960 INFO GENERATE SUBMISSION FILE
2017-06-07 20:59:59,968 INFO detection result exported to ../../output/kaggle-sea-lion/20/1/submission.csv
2017-06-07 20:59:59,971 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/41.jpg (48418.884 ms)






    












    



2017-06-07 21:00:00,049 INFO image 48
2017-06-07 21:00:00,051 INFO total detections: 87
2017-06-07 21:00:00,052 INFO class detections: [87, 0, 0, 0, 0]
2017-06-07 21:00:00,054 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/48.jpg (64507.047 ms)
2017-06-07 21:00:00,055 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/50.jpg...






    



(400, 4992, 3)






    



2017-06-07 21:00:00,302 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 21:00:00,305 INFO > [started] sliding_window...






    



399/400 [=========================>] 99% 35s remaining=0s sliding windoww





    



2017-06-07 21:00:36,074 INFO > [done]    sliding_window (35768.560 ms)
2017-06-07 21:00:36,075 INFO > [started] non_max_suppression. boxes=0...
2017-06-07 21:00:36,077 INFO > [done]    non_max_suppression. boxes=0 (1.961 ms)






    












    



2017-06-07 21:00:38,182 INFO image 50
2017-06-07 21:00:38,185 INFO total detections: 0
2017-06-07 21:00:38,186 INFO class detections: [0, 0, 0, 0, 0]
2017-06-07 21:00:38,188 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/50.jpg (38132.503 ms)
2017-06-07 21:00:38,203 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/42.jpg...






    



(400, 4992, 3)






    



2017-06-07 21:00:38,462 INFO pyramid layer=0 image=(400, 4992, 3) scale=1
2017-06-07 21:00:38,464 INFO > [started] sliding_window...






    



342/400 [=====================>....] 85% 34s remaining=5s sliding windoww





    



2017-06-07 21:01:13,393 INFO > [started] non_max_suppression. boxes=1000102...
2017-06-07 21:01:17,099 INFO > [done]    non_max_suppression. boxes=1000102 (3706.639 ms)






    



399/400 [=========================>] 99% 40s remaining=0s sliding window





    



2017-06-07 21:01:18,943 INFO > [done]    sliding_window (40479.627 ms)
2017-06-07 21:01:18,946 INFO > [started] non_max_suppression. boxes=378548...
2017-06-07 21:01:20,295 INFO > [done]    non_max_suppression. boxes=378548 (1349.048 ms)






    












    



2017-06-07 21:01:22,259 INFO image 42
2017-06-07 21:01:22,261 INFO total detections: 43
2017-06-07 21:01:22,263 INFO class detections: [43, 0, 0, 0, 0]
2017-06-07 21:01:22,265 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/42.jpg (44062.010 ms)
2017-06-07 21:01:22,267 INFO GENERATE SUBMISSION FILE
2017-06-07 21:01:22,275 INFO detection result exported to ../../output/kaggle-sea-lion/20/2/submission.csv
2017-06-07 21:01:22,279 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/42.jpg (44075.776 ms)
2017-06-07 21:01:22,414 INFO ==== ALL DONE ====

Combine all shard results



In [7]:

    
logger.info('COMBINING ALL SHARD RESULTS INTO ONE SUBMISSION FILE')

total_detected_lions = np.array([])
for sd in shard_group.shard_dirs():
    df = pd.read_csv(sd + 'submission.csv')
    if(len(total_detected_lions)>0):
        total_detected_lions = np.concatenate((total_detected_lions, df.as_matrix()))
    else:
        total_detected_lions = df.as_matrix()

submission_file = OUTPUT_DIR + 'submission-merged.csv'
df = pd.DataFrame(total_detected_lions, columns=('test_id','adult_males','subadult_males','adult_females','juveniles','pups'))
df = df.sort_values('test_id')
df.to_csv(submission_file, index=False)
logger.info('submission file merged')









    



2017-06-07 21:01:22,425 INFO COMBINING ALL SHARD RESULTS INTO ONE SUBMISSION FILE
2017-06-07 21:01:22,467 INFO submission file merged

Fazendo teste



In [ ]:



In [ ]: