notebook.community

Edit and run

Generate competition submission





In [1]:



    


INPUT_DIR = '../../input/kaggle-sea-lion/'
OUTPUT_DIR = '../../output/kaggle-sea-lion/22/'

LABEL_DIMS = (6,)
#classes meta data in format [image dimensions,model weights path,sliding window step]
CLASSES_CONFIG = [
    [(84,84,3),INPUT_DIR + '05/weights-medium1-class0-84x84.h5', (21,21)],
    [(84,84,3),INPUT_DIR + '05/weights-medium1-class1-84x84.h5', (21,21)],
    [(56,56,3),INPUT_DIR + '05/weights-medium1-class2-56x56.h5', (19,19)],
    [(42,42,3),INPUT_DIR + '05/weights-medium1-class3-42x42.h5', (9,9)],
    [(26,26,3),INPUT_DIR + '05/weights-medium1-class4-21x21.h5', (8,8)]
]
#CLASSES_CONFIG = [
#    [(84,84,3),INPUT_DIR + '05/weights-medium1-class0-84x84.h5', (21,21)],
#    [(84,84,3),INPUT_DIR + '05/weights-medium1-class0-84x84.h5', (21,21)],
#    [(84,84,3),INPUT_DIR + '05/weights-medium1-class0-84x84.h5', (21,21)],
#    [(84,84,3),INPUT_DIR + '05/weights-medium1-class0-84x84.h5', (21,21)],
#    [(84,84,3),INPUT_DIR + '05/weights-medium1-class0-84x84.h5', (21,21)]
#]
DEBUG = True
IMAGE_SLICE = slice(1500,1600)
#IMAGE_SLICE = slice(0,99999)

NR_SHARDS = 4
RECREATE_OUTPUT = True



    











In [2]:



    


%matplotlib inline
import numpy as np
import pandas as pd
import h5py
import matplotlib.pyplot as plt
import sklearn
import os
import glob
import cv2
from multiprocessing import Pool
import multiprocessing

from modules.logging import logger
import modules.utils as utils
from modules.utils import Timer
import modules.logging as logging
import modules.cnn as cnn
import modules.lions as lions
import modules.shards as shards
import modules.objectdetect as objectdetect



    


















    






Using TensorFlow backend.

Classify and count lions on each image

Find lions on all images





In [3]:



    


#from MismatchedTrainImages.txt
MISMATCHED = [3, 7, 9, 21, 30, 34, 71, 81, 89, 97, 151, 184, 215, 234, 242, 268, 290, 311, 331, 344, 380, 384, 406, 421, 469, 475, 490, 499, 507, 530, 531, 605, 607, 614, 621, 638, 644, 687, 712, 721, 767, 779, 781, 794, 800, 811, 839, 840, 869, 882, 901, 903, 905, 909, 913, 927, 946]
FORCE_IMAGES = [42]
#FORCE_IMAGES = None



    











In [4]:



    


class ClassClassifier:
    def __init__(self, model):
        self.model = model
        
    def evaluate_region(self, image):
        y_pred = self.model.predict(np.array([image]))
        ylp = utils.onehot_to_label(np.array(y_pred))
        return y_pred[0][ylp[0]], ylp[0]



    











In [5]:



    


def process_shard_lions(shard_group, shard_id):
    t = Timer('PROCESSING SHARD {}'.format(shard_id))

    output_dir = shard_group.shard_dir(shard_id)
    logging.setup_file_logger(output_dir + 'out.log')
    
    logger.info('Load CNN models ' + str(CLASSES_CONFIG))
    class_eval_functions = []
    for class_config in CLASSES_CONFIG:
        #lion simple cnn forward time: ~10ms
        model = lions.convnet_medium1_lion_keras(class_config[0])
        model.load_weights(class_config[1])
        classifier = ClassClassifier(model)
        class_eval_functions.append(classifier.evaluate_region)
    
    #DECLARE FUNCTIONS INSIDE PROCESS INSTANCE
    """ Returns (score, label) """
    def evaluate_region_all_classes(region_img):
        y_pred = model.predict(np.array([region_img]))
        ylp = utils.onehot_to_label(np.array(y_pred))
        return y_pred[0][ylp[0]], ylp[0]

    def detection_to_colortext(detection):
        score = detection[4]
        text = str(int(detection[5])) + ' ' + '{0:.2f}'.format(score)
        c = int(score*255)
        return (0,0,c), text
    
    def detect_lions(image, evaluate_function, sliding_step, window_size):
        #search for lions
        region_generator = objectdetect.sliding_window_generator(image, step=sliding_step, window=window_size, pyramid_max_layers=1)
        detections, imgs = objectdetect.evaluate_regions(region_generator, evaluate_function, filter_score_min=0.97, 
                                                         filter_labels=(0,1,2,3,4), apply_non_max_suppression=True, 
                                                         supression_overlap_threshold=0.3, threads=None)

        if(DEBUG):
            img2 = image.copy()
            objectdetect.draw_detections(detections, img2, detection_to_colortext)
            utils.show_image(img2, size=60, is_bgr=True)
    #        patches = objectdetect.extract_patches(detections, img)
    #        utils.show_images(patches[0:50], size=2, cols=10, is_bgr=True)   

        return detections

    if(shard_group.shard_done(shard_id)):
        logger.warning('shard {} already processed. Skipping'.format(shard_id))

    else:
        image_paths = shard_group.shard_items(shard_id)

        total_lions_count = []

        for image_path in image_paths:
            base = os.path.basename(image_path)
            fn = os.path.splitext(base)
            if(fn[1]!='.jpg'):
                logger.info('ignoring non jpg image. filename=' + base)
                continue

            train_id = int(fn[0])

            if(FORCE_IMAGES!=None and train_id not in FORCE_IMAGES):
                continue

            t = Timer('processing photo ' + image_path)
            image_raw = cv2.imread(image_path)
            image_raw = image_raw[IMAGE_SLICE]

            
            #detect lions on each classifier model
            detected_lions = None
            for i,eval_function in enumerate(class_eval_functions):
                logger.info('SEARCHING CLASS ' + str(i) + '...')
                window_size = CLASSES_CONFIG[i][0]
                step_size = CLASSES_CONFIG[i][2]
                detections = detect_lions(image_raw, eval_function, step_size, window_size)
                if(detected_lions==None):
                    detected_lions = detections
                else:
                    detected_lions = np.concatenate((detected_lions, np.array(detections)))

            detected_lions = detected_lions.tolist()
                
            logger.info('performing nms')
            print(len(detected_lions))
            detected_lions = objectdetect.non_maxima_suppression(detected_lions, overlap_threshold=0.15)
            print(len(detected_lions))

            if(DEBUG):
                img2 = image_raw.copy()
                objectdetect.draw_detections(detected_lions, img2, detection_to_colortext)
                utils.show_image(img2, size=60, is_bgr=True)

                
            #calculate class distribution
            lions_count = np.zeros(LABEL_DIMS[0]-1, dtype='int')
            for detection in detected_lions:
                label = int(detection[5])
                lions_count[label] += 1 
            lions_count = lions_count.tolist()
            
            logger.info('image ' + str(train_id))
            logger.info('total detections: ' + str(np.sum(np.array(lions_count))))
            logger.info('class detections: ' + str(lions_count))
            total_lions_count += [[train_id] + lions_count]

            print('total_lions_count')
            print(total_lions_count)
            t.stop()

        logger.info('GENERATE SUBMISSION FILE')
        submission_file = output_dir + 'submission.csv'
        df = pd.DataFrame(total_lions_count, columns=('test_id','adult_males','subadult_males','adult_females','juveniles','pups'))
        df.to_csv(submission_file, index=False)

        logger.info('detection result exported to ' + submission_file)        
        shard_group.mark_done(shard_id)

        t.stop()



    











In [6]:



    


logger.info('==== PROCESSING SHARDS IN PARALLEL ====')

logger.info('preparing shards')
images_dir = INPUT_DIR + "Train/"
image_paths = [images_dir+n for n in os.listdir(images_dir)]

shard_group = shards.ShardGroup(image_paths, NR_SHARDS, OUTPUT_DIR, recreate_shards_dir=RECREATE_OUTPUT)



    


















    






2017-06-09 00:16:38,501 INFO ==== PROCESSING SHARDS IN PARALLEL ====
2017-06-09 00:16:38,503 INFO preparing shards

















In [7]:



    


shard_group.start_processing(process_shard_lions, shard_ids=None, threads=-1)
logger.info('==== ALL DONE ====')



    


















    






2017-06-09 00:16:38,533 INFO Using 4 parallel tasks
2017-06-09 00:16:40,563 INFO > [started] PROCESSING SHARD 4...
2017-06-09 00:16:40,574 INFO Load CNN models [[(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)]]
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, kernel_initializer="glorot_uniform", activation="softmax")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-09 00:16:43,563 INFO > [started] PROCESSING SHARD 3...
2017-06-09 00:16:43,578 INFO Load CNN models [[(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)]]
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, kernel_initializer="glorot_uniform", activation="softmax")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-09 00:16:44,434 INFO found 2 items for shard 4
2017-06-09 00:16:44,437 INFO ignoring non jpg image. filename=MismatchedTrainImages.txt
2017-06-09 00:16:44,439 INFO ignoring non jpg image. filename=.ipynb_checkpoints
2017-06-09 00:16:44,441 INFO GENERATE SUBMISSION FILE
2017-06-09 00:16:44,449 INFO detection result exported to ../../output/kaggle-sea-lion/22/4/submission.csv
2017-06-09 00:16:44,455 INFO > [done]    PROCESSING SHARD 4 (3892.419 ms)
2017-06-09 00:16:45,567 INFO > [started] PROCESSING SHARD 2...
2017-06-09 00:16:45,578 INFO Load CNN models [[(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)]]
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, kernel_initializer="glorot_uniform", activation="softmax")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-09 00:16:46,566 INFO > [started] PROCESSING SHARD 1...
2017-06-09 00:16:46,583 INFO Load CNN models [[(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)], [(84, 84, 3), '../../input/kaggle-sea-lion/05/weights-medium1-class0-84x84.h5', (21, 21)]]
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:69: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(64, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:71: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(128, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:73: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(256, (3, 3), kernel_initializer="glorot_uniform", padding="same", activation="relu")`
  model.add(convolutional.Conv2D(256, (3, 3), activation='relu', padding='same', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:78: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:80: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(1024, kernel_initializer="glorot_uniform", activation="relu")`
  model.add(core.Dense(1024, activation='relu', init='glorot_uniform'))
/notebooks/datascience-snippets/kaggle-sea-lion/modules/lions.py:82: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(6, kernel_initializer="glorot_uniform", activation="softmax")`
  model.add(core.Dense(6, activation='softmax', init='glorot_uniform'))
2017-06-09 00:16:48,350 INFO found 3 items for shard 3
2017-06-09 00:16:48,355 INFO ignoring non jpg image. filename=train.csv
2017-06-09 00:16:48,358 INFO GENERATE SUBMISSION FILE
2017-06-09 00:16:48,368 INFO detection result exported to ../../output/kaggle-sea-lion/22/3/submission.csv
2017-06-09 00:16:48,377 INFO > [done]    PROCESSING SHARD 3 (4813.430 ms)
2017-06-09 00:16:50,568 INFO found 5 items for shard 2
2017-06-09 00:16:50,570 INFO > [started] processing photo ../../input/kaggle-sea-lion/Train/42.jpg...
2017-06-09 00:16:50,917 INFO SEARCHING CLASS 0...










    






(100, 4992, 3)










    






2017-06-09 00:16:50,923 INFO pyramid layer=0 image=(100, 4992, 3) scale=1
2017-06-09 00:16:50,926 INFO > [started] sliding_window...










    






0/100 [>.........................] 0% 0s sliding window









    






2017-06-09 00:16:51,454 INFO found 3 items for shard 1
2017-06-09 00:16:51,455 INFO GENERATE SUBMISSION FILE
2017-06-09 00:16:51,464 INFO detection result exported to ../../output/kaggle-sea-lion/22/1/submission.csv
2017-06-09 00:16:51,469 INFO > [done]    PROCESSING SHARD 1 (4903.450 ms)










    






84/100 [=====================>....] 84% 4s remaining=0s sliding windoww









    






2017-06-09 00:16:55,738 INFO > [done]    sliding_window (4812.018 ms)
2017-06-09 00:16:55,742 INFO > [started] non_max_suppression. boxes=12982...
2017-06-09 00:16:55,808 INFO > [done]    non_max_suppression. boxes=12982 (65.668 ms)










    

















    






2017-06-09 00:16:57,004 INFO SEARCHING CLASS 1...










    






(100, 4992, 3)










    






2017-06-09 00:16:57,008 INFO pyramid layer=0 image=(100, 4992, 3) scale=1
2017-06-09 00:16:57,010 INFO > [started] sliding_window...










    






84/100 [=====================>....] 84% 5s remaining=1s sliding windoww









    






2017-06-09 00:17:02,329 INFO > [done]    sliding_window (5318.720 ms)
2017-06-09 00:17:02,331 INFO > [started] non_max_suppression. boxes=12982...
2017-06-09 00:17:02,448 INFO > [done]    non_max_suppression. boxes=12982 (116.534 ms)










    

















    






/usr/local/lib/python3.4/dist-packages/ipykernel/__main__.py:77: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.
2017-06-09 00:17:03,644 INFO SEARCHING CLASS 2...










    






(100, 4992, 3)










    






2017-06-09 00:17:03,647 INFO pyramid layer=0 image=(100, 4992, 3) scale=1
2017-06-09 00:17:03,650 INFO > [started] sliding_window...










    






84/100 [=====================>....] 84% 5s remaining=0s sliding windoww









    






2017-06-09 00:17:08,881 INFO > [done]    sliding_window (5231.443 ms)
2017-06-09 00:17:08,884 INFO > [started] non_max_suppression. boxes=12982...
2017-06-09 00:17:08,947 INFO > [done]    non_max_suppression. boxes=12982 (62.583 ms)










    

















    






2017-06-09 00:17:10,151 INFO SEARCHING CLASS 3...










    






(100, 4992, 3)










    






2017-06-09 00:17:10,155 INFO pyramid layer=0 image=(100, 4992, 3) scale=1
2017-06-09 00:17:10,157 INFO > [started] sliding_window...










    






84/100 [=====================>....] 84% 4s remaining=0s sliding windoww









    






2017-06-09 00:17:14,772 INFO > [done]    sliding_window (4614.666 ms)
2017-06-09 00:17:14,774 INFO > [started] non_max_suppression. boxes=12982...
2017-06-09 00:17:14,837 INFO > [done]    non_max_suppression. boxes=12982 (63.436 ms)










    

















    






2017-06-09 00:17:16,052 INFO SEARCHING CLASS 4...










    






(100, 4992, 3)










    






2017-06-09 00:17:16,057 INFO pyramid layer=0 image=(100, 4992, 3) scale=1
2017-06-09 00:17:16,060 INFO > [started] sliding_window...










    






84/100 [=====================>....] 84% 4s remaining=0s sliding windoww









    






2017-06-09 00:17:20,988 INFO > [done]    sliding_window (4928.020 ms)
2017-06-09 00:17:20,990 INFO > [started] non_max_suppression. boxes=12982...
2017-06-09 00:17:21,058 INFO > [done]    non_max_suppression. boxes=12982 (68.058 ms)










    

















    






2017-06-09 00:17:22,251 INFO performing nms










    






20
3










    

















    






2017-06-09 00:17:23,485 INFO image 42
2017-06-09 00:17:23,487 INFO total detections: 3
2017-06-09 00:17:23,490 INFO class detections: [3, 0, 0, 0, 0]










    






total_lions_count
[[42, 3, 0, 0, 0, 0]]










    






2017-06-09 00:17:23,495 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/42.jpg (32925.123 ms)
2017-06-09 00:17:23,498 INFO GENERATE SUBMISSION FILE
2017-06-09 00:17:23,506 INFO detection result exported to ../../output/kaggle-sea-lion/22/2/submission.csv
2017-06-09 00:17:23,510 INFO > [done]    processing photo ../../input/kaggle-sea-lion/Train/42.jpg (32940.347 ms)
2017-06-09 00:17:23,798 INFO ==== ALL DONE ====

Combine all shard results





In [8]:



    


logger.info('COMBINING ALL SHARD RESULTS INTO ONE SUBMISSION FILE')

total_detected_lions = np.array([])
for sd in shard_group.shard_dirs():
    df = pd.read_csv(sd + 'submission.csv')
    if(len(total_detected_lions)>0):
        total_detected_lions = np.concatenate((total_detected_lions, df.as_matrix()))
    else:
        total_detected_lions = df.as_matrix()

submission_file = OUTPUT_DIR + 'submission-merged.csv'
df = pd.DataFrame(total_detected_lions, columns=('test_id','adult_males','subadult_males','adult_females','juveniles','pups'))
df = df.sort_values('test_id')
df.to_csv(submission_file, index=False)
logger.info('submission file merged')



    


















    






2017-06-09 00:17:23,807 INFO COMBINING ALL SHARD RESULTS INTO ONE SUBMISSION FILE
2017-06-09 00:17:23,836 INFO submission file merged

Fazendo teste





In [ ]:



    






    











In [ ]:

Content source: flaviostutz/datascience-snippets

Similar notebooks:

notebook.community | gallery | about