In [6]:

    

from subprocess import Popen, PIPE, STDOUT
import re
import my_io
import time
import sys


    

In [4]:

    

ITERS_PER_EPOCH = 75

train_caffe_cmd = ['~/documents/caffe/build/tools/caffe',
                   'train',
                   '--solver=/media/raid_arr/data/ndsb/config/solver.prototxt',
                   ]
                   
p = Popen(train_caffe_cmd, stdout = PIPE, 
        stderr = STDOUT, shell = True)
while True:
    line = p.stdout.readline()
    if not line: break
    ii_re = re.search(r'Iteration (\d+), loss', line)
    if ii_re and int(ii_re.groups()[0]) % ITERS_PER_EPOCH == 0:
        print 'Approx 1 epoch has passed - time to re-augment'
        my_io.create_aug_db(orig_db='/dev/shm/train0_lmdb',
                            aug_db='/dev/shm/train0_aug_lmdb')


    

In [16]:

    

ITERS_PER_EPOCH = 75
with open('/tmp/my_caffe_log.txt', 'r') as f:
    while True: 
        line = f.readline()
        if not line:
            break
    while True:
        line = f.readline()
        if not line: 
            time.sleep(5)
        ii_re = re.search(r'Iteration (\d+), loss', line)
        if ii_re and int(ii_re.groups()[0]) % ITERS_PER_EPOCH == 0:
            print 'Approx 1 epoch has passed - time to re-augment'
            sys.stdout.flush()
            my_io.create_aug_db()

# my_io.create_aug_db()


    

    




Approx 1 epoch has passed - time to re-augment
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /dev/shm/train0_aug_lmdb took 12.7031600475 sec
Approx 1 epoch has passed - time to re-augment
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /dev/shm/train0_aug_lmdb took 12.6938509941 sec
Approx 1 epoch has passed - time to re-augment
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /dev/shm/train0_aug_lmdb took 12.7027108669 sec
Approx 1 epoch has passed - time to re-augment
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /dev/shm/train0_aug_lmdb took 12.716848135 sec






    




---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-16-262b0d3567b1> in <module>()
      8         line = f.readline()
      9         if not line:
---> 10             time.sleep(5)
     11         ii_re = re.search(r'Iteration (\d+), loss', line)
     12         if ii_re and int(ii_re.groups()[0]) % ITERS_PER_EPOCH == 0:

KeyboardInterrupt: 

In [15]:

    

my_io.create_aug_db()


    

    




Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /dev/shm/train0_aug_lmdb took 12.6841449738 sec