notebook.community

Edit and run



In [18]:

    
import os                                                                                               
import subprocess
import my_io
import threading
NDSB_DIR = '/media/raid_arr/data/ndsb/config'
# TRAIN_SCRIPT = os.path.join(NDSB_DIR, 'train_pl.sh')
# RESUME_SCRIPT = os.path.join(NDSB_DIR, 'resume_training_pl.sh')
SOLVER = os.path.join(NDSB_DIR, 'solver.prototxt')
NET = os.path.join(NDSB_DIR, 'train_val.prototxt')
CAFFE = '/afs/ee.cooper.edu/user/t/a/tam8/documents/caffe/build/tools/caffe'
MODELS_DIR = '/media/raid_arr/data/ndsb/models'
snapshot_prefix = 'alexmod_bruteaug_fold0_iter_'
MAX_ITER = 100000    # global max (not per step)
STEP = 100



In [19]:

    
def write_max_iter_to_solver(max_iter, f_path=SOLVER):
    with open(f_path, 'r') as f:
    # read a list of lines into data
        f_data = f.readlines()

    # Change the line with the PL loss weight
    line_n = 8    # The line number we're going to replace
    new_line = 'max_iter: ' + str(max_iter) + '\n'
    f_data[line_n] = new_line

    # and write everything back
    with open(f_path, 'w') as f:
        f.writelines(f_data)
    return f_path



In [20]:

    
snap_name = lambda n_iter: os.path.join(MODELS_DIR,
            snapshot_prefix + str(n_iter) + '.solverstate')

call_start = lambda sol=SOLVER: subprocess.call(
            [CAFFE, 'train', '--solver=' + sol])
call_resume = lambda snap, sol=SOLVER: subprocess.call(
            [CAFFE, 'train', '--solver=' + sol, '--snapshot=' + snap])



In [22]:

    
last_saved_iter = max({int(os.path.splitext(f)[0].rsplit('_', 1)[1]) 
                       for f in next(os.walk(MODELS_DIR))[2]}.union({0}))  

for ii in range(last_saved_iter, MAX_ITER+1, STEP):
    write_max_iter_to_solver(ii + STEP) # make caffe stop at the next step
    print 'ITER:\t', ii
    
    subprocess.call(['rm', '-rf', '/dev/shm/train0_aug_lvl'])
    subprocess.call(['cp', '-rf',
                     '/media/raid_arr/tmp/train0_aug_lvl',
                     '/dev/shm/train0_aug_lvl'])
    # Start augmenting in another thread while caffe runs
    aug_thread = threading.Thread(target=my_io.create_aug_lvl)
    aug_thread.start()
    
    if ii == 0:
        print 'Starting new train'
        call_start()
        subprocess.call(['cp', '/tmp/caffe.INFO', 
                         '/tmp/my_caffe_log.txt'])
    else:
        call_resume(snap_name(ii))
        subprocess.call(['cat /tmp/caffe.INFO >> /tmp/my_caffe_log.txt'], shell=True)
print 'DONE'









    



ITER:	400
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 23.3951058388 sec
ITER:	500
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 31.0545511246 sec
ITER:	600
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.6482019424 sec
ITER:	700
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.3256220818 sec
ITER:	800
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 22.3219180107 sec
ITER:	900
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 20.3777880669 sec
ITER:	1000
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.5033640862 sec
ITER:	1100
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.4882810116 sec
ITER:	1200
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 20.5696358681 sec
ITER:	1300
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.5119621754 sec
ITER:	1400
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.4803161621 sec
ITER:	1500
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 22.8650701046 sec
ITER:	1600
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 22.8902490139 sec
ITER:	1700
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.2877149582 sec
ITER:	1800
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.519190073 sec
ITER:	1900
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.2639980316 sec
ITER:	2000
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.2895431519 sec
ITER:	2100
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.1144919395 sec
ITER:	2200
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.0925729275 sec
ITER:	2300
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.2371640205 sec
ITER:	2400
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.35729599 sec
ITER:	2500
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 32.4463670254 sec
ITER:	2600
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.6218709946 sec
ITER:	2700
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.2813549042 sec
ITER:	2800
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 22.3609278202 sec
ITER:	2900
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.2305321693 sec
ITER:	3000
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 20.8340039253 sec
ITER:	3100
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.6067421436 sec
ITER:	3200
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.650067091 sec
ITER:	3300
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 21.7929639816 sec
ITER:	3400
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.1101200581 sec
ITER:	3500
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 32.6959319115 sec
ITER:	3600
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.6990809441 sec
ITER:	3700
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 21.8797810078 sec
ITER:	3800
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.5075337887 sec
ITER:	3900
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 15.3633520603 sec
ITER:	4000
Processed: 0
Processed: 5000
Processed: 10000
Processed: 15000
Processed: 20000
Conversion of images from /dev/shm/train0_lmdb to /media/raid_arr/tmp/train0_aug_lvl took 21.4040081501 sec






    



---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-22-c1286cac557a> in <module>()
     20                          '/tmp/my_caffe_log.txt'])
     21     else:
---> 22         call_resume(snap_name(ii))
     23         subprocess.call(['cat /tmp/caffe.INFO >> /tmp/my_caffe_log.txt'], shell=True)
     24 print 'DONE'

<ipython-input-20-9ce95f5e1a4b> in <lambda>(snap, sol)
      5             [CAFFE, 'train', '--solver=' + sol])
      6 call_resume = lambda snap, sol=SOLVER: subprocess.call(
----> 7             [CAFFE, 'train', '--solver=' + sol, '--snapshot=' + snap])

/usr/lib/python2.7/subprocess.pyc in call(*popenargs, **kwargs)
    520     retcode = call(["ls", "-l"])
    521     """
--> 522     return Popen(*popenargs, **kwargs).wait()
    523 
    524 

/usr/lib/python2.7/subprocess.pyc in wait(self)
   1374             while self.returncode is None:
   1375                 try:
-> 1376                     pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
   1377                 except OSError as e:
   1378                     if e.errno != errno.ECHILD:

/usr/lib/python2.7/subprocess.pyc in _eintr_retry_call(func, *args)
    474     while True:
    475         try:
--> 476             return func(*args)
    477         except (OSError, IOError) as e:
    478             if e.errno == errno.EINTR:

KeyboardInterrupt:



In [9]:

    
range(last_saved_iter, MAX_ITER+1, STEP)