In [5]:

    

import os
import subprocess as sub
import sys
import time
import numpy as np
import my_io

BUFF_SIZE = 20
BUFFER_PATH = '/media/raid_arr/tmp/aug_buffer/'
TRAIN_FOLD_TXT = '/media/raid_arr/data/ndsb/folds/train0.txt'
CHECK_INTERVAL = 5


    

In [6]:

    

def add_to_buffer(im_paths, buffer_dir, verbose=False):
    db_name = str(int(time.time()))
    db_out_path = os.path.join(buffer_dir, db_name)
    np.random.shuffle(train_fold_paths)
    my_io.multi_extract(train_fold_paths, db_out_path, backend='leveldb',
                    perturb=True, verbose=True)
    if verbose:
        print 'Created db:', db_out_path


    

In [7]:

    

def act_buffer(im_paths, buffer_dir=BUFFER_PATH, buff_size=BUFF_SIZE, check_int=CHECK_INTERVAL, verbose=False):
    while True:
        if len(os.walk(BUFFER_PATH).next()[1]) < buff_size:
            add_to_buffer(im_paths, buffer_dir, verbose=verbose)
        else:
            if verbose:
                print 'Waiting', check_int
            time.sleep(check_int)


    

In [8]:

    

import warnings
warnings.filterwarnings('ignore')
train_fold_paths = np.loadtxt(TRAIN_FOLD_TXT, delimiter='\t', dtype=str)[:, 0]
act_buffer(train_fold_paths, 
           buffer_dir=BUFFER_PATH, 
           buff_size=BUFF_SIZE, 
           check_int=CHECK_INTERVAL)


    

    




Multiproc extraction: 87.8194899559
Multiproc extraction: 85.1782288551
Multiproc extraction: 85.0888741016
Multiproc extraction: 84.7708349228
Multiproc extraction: 84.9402341843
Multiproc extraction: 86.4700348377
Multiproc extraction: 87.3924582005
Multiproc extraction: 85.3525590897
Multiproc extraction: 86.4573168755
Multiproc extraction: 84.9750151634
Multiproc extraction: 107.704458952






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-8-d8915c54a96e> in <module>()
      5            buffer_dir=BUFFER_PATH,
      6            buff_size=BUFF_SIZE,
----> 7            check_int=CHECK_INTERVAL)

<ipython-input-7-12edd2a2fc2e> in act_buffer(im_paths, buffer_dir, buff_size, check_int, verbose)
      6             if verbose:
      7                 print 'Waiting', check_int
----> 8             time.sleep(check_int)

KeyboardInterrupt: