In [1]:
import sys
sys.path.append("../../xcorr")

In [33]:
DB.close()

import xcorr_db
import importlib
importlib.reload(xcorr_db)


Out[33]:
<module 'xcorr_db' from '../../xcorr/xcorr_db.py'>

In [34]:
DB = xcorr_db.setup_db('./initial.db')

In [35]:
hpo_id = DB.insert_hpo_record(2)


---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-35-511998c74e9d> in <module>
----> 1 hpo_id = DB.insert_hpo_record(2)

~/Documents/repos/Supervisor/workflows/xcorr/xcorr_db.py in insert_hpo_record(self, record_id)
     52         with self.conn:
     53             sql = 'insert into hpos (xcorr_record_id, time) values(?, ?)'
---> 54             self.cursor.execute(sql, record_id, ts)
     55             # hpo_id = self.insert(table='hpos', names=['xcorr_record_id', 'time'],
     56             #     values = [str(record_id), q(ts)])

~/Documents/repos/Supervisor/workflows/xcorr/xcorr_db.py in insert(self, table, names, values)
    131         return self.study_id2name, self.study_name2id
    132 
--> 133     def insert(self, table, names, values):
    134         """ Do a SQL insert """
    135         names_tpl  = sql_tuple(names)

~/Documents/repos/Supervisor/workflows/xcorr/xcorr_db.py in sql_tuple(L)
    175     result = ""
    176     result += "("
--> 177     result += ",".join(L)
    178     result += ")"
    179     return result

TypeError: sequence item 0: expected str instance, int found

In [131]:
import pandas as pd
import csv
import numpy as np
import json

Generate Stats From hpos.txt

Each hpo writes an hpo.txt with hpo_id, params, run_dir, ts, and val loss


In [209]:
df = pd.read_csv('~/Documents/results/cp1/non_nci_hpo_log/hpos.txt', sep="|", header=None, names=["i", "hpo_id", "params", "run_dir", "ts", "val_loss"])

In [212]:
df.groupby("hpo_id")['val_loss'].agg([np.min, np.max, np.mean, np.std])


Out[212]:
amin amax mean std
hpo_id
1 0.325794 2.205444e+34 3.968941e+31 9.029660e+32
2 0.278948 1.036991e+31 1.955343e+28 3.718788e+29
3 0.321078 5.450333e+32 7.536106e+29 1.865691e+31
4 0.269695 3.885900e+33 4.450476e+30 1.295403e+32
5 0.326156 1.407330e+32 1.564757e+29 4.691097e+30
6 0.325993 5.586196e+33 6.231631e+30 1.862063e+32
7 0.332956 2.311167e+30 5.317878e+27 8.994357e+28
8 0.343334 5.819235e+31 9.109562e+28 1.979917e+30
9 0.333361 1.440266e+30 4.994766e+27 6.875268e+28
10 0.280865 1.324509e+31 3.563354e+28 6.101971e+29
11 0.285089 6.691469e+30 7.697821e+27 2.230753e+29
12 0.295384 3.726767e+34 4.183554e+31 1.242292e+33
13 0.319555 2.266933e+31 7.312421e+28 1.152551e+30
14 0.279993 6.867474e+33 7.747196e+30 2.289245e+32
15 0.289928 1.351673e+31 2.596114e+28 5.137327e+29
16 0.294270 4.708584e+31 5.354296e+28 1.569902e+30
17 0.273182 5.815590e+33 9.600886e+30 2.130202e+32
18 0.321369 2.091420e+30 1.061623e+28 1.180208e+29
19 0.296538 5.278538e+33 1.205307e+31 2.080988e+32
20 0.330869 1.100065e+33 1.226361e+30 3.666880e+31

In [161]:
n = 10
smallest = df.groupby('hpo_id')['val_loss'].nsmallest(n)
best_n = df.iloc[smallest.index.get_level_values(1), :]
#best_n.to_csv('~/Documents/results/cp1/best_{}_nci.txt'.format(n), sep="|", index=False)
params = '/home/nick/Documents/results/cp1/best_{}_nci_params.txt'.format(n)
# Write out the best n parameters
best_n[['params']].to_csv(params, index=False, 
                        header=False, quoting=csv.QUOTE_NONE, sep='|')

In [102]:
pd.__version__


Out[102]:
'0.22.0'

In [141]:
# stats for the best n
best_n.groupby("hpo_id")['val_loss'].agg([np.min, np.max, np.mean, np.std])


Out[141]:
amin amax mean std
hpo_id
1 0.201724 0.206538 0.203615 0.001687
2 0.179843 0.188933 0.185792 0.003026
3 0.186955 0.196299 0.191846 0.003553
4 0.184138 0.194240 0.190929 0.003104
5 0.184527 0.196454 0.192813 0.003435

Generate upf -- one json param dict per line -- file from the best n runs


In [165]:
from os import path

with open(params) as f_in:
    lines = f_in.readlines()
    
upf = '/home/nick/Documents/results/cp1/best_{}_nci_params_upf.txt'.format(n)
with open(upf, 'w') as f_out:
    for line in lines:
        j = json.loads(line)
        train_sources = j['train_sources']
        if 'cell_feature_subset_path' in j and train_sources == 'NCI60':
            fsp = path.basename(j['cell_feature_subset_path'])
            train = '{}_train.h5'.format(fsp[:fsp.find('_features')])
            j['use_exported_data'] = '/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/cache/{}'.format(train)
            
            
        del j['save_path']
        j['epochs'] = 100
        f_out.write('{}\n'.format(json.dumps(j)))

Create upf for creating test, train generated data


In [189]:
js = """{{"batch_size": 6144, "train_sources": "{}", "preprocess_rnaseq": "combat", "gpus": "0 1 2 3 4 5", "cell_feature_subset_path": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/xcorr_data/{}_{}_2000_1000_features.txt", "export_data": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/cache/{}_{}_2000_1000_{}.h5", "no_feature_source": true, "no_response_source": true, "cp": true}}"""



studies = ['CCLE', 'CTRP', 'gCSI', 'GDSC']
for s1 in studies:
    for s2 in studies:
        if s1 != s2:
            j1 = js.format(s1, s1, s2, s1, s2, 'train')
            j2 = js.format(s2, s1, s2, s1, s2, 'test')
            #print(j1)
            #print(j2)
            
for s1 in studies:
    s2 = 'NCI60'
    j = js.format(s2, s1, s2, s1, s2, 'test')
    print(j)


{"batch_size": 6144, "train_sources": "NCI60", "preprocess_rnaseq": "combat", "gpus": "0 1 2 3 4 5", "cell_feature_subset_path": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/xcorr_data/CCLE_NCI60_2000_1000_features.txt", "export_data": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/cache/CCLE_NCI60_2000_1000_test.h5", "no_feature_source": true, "no_response_source": true, "cp": true}
{"batch_size": 6144, "train_sources": "NCI60", "preprocess_rnaseq": "combat", "gpus": "0 1 2 3 4 5", "cell_feature_subset_path": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/xcorr_data/CTRP_NCI60_2000_1000_features.txt", "export_data": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/cache/CTRP_NCI60_2000_1000_test.h5", "no_feature_source": true, "no_response_source": true, "cp": true}
{"batch_size": 6144, "train_sources": "NCI60", "preprocess_rnaseq": "combat", "gpus": "0 1 2 3 4 5", "cell_feature_subset_path": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/xcorr_data/gCSI_NCI60_2000_1000_features.txt", "export_data": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/cache/gCSI_NCI60_2000_1000_test.h5", "no_feature_source": true, "no_response_source": true, "cp": true}
{"batch_size": 6144, "train_sources": "NCI60", "preprocess_rnaseq": "combat", "gpus": "0 1 2 3 4 5", "cell_feature_subset_path": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/xcorr_data/GDSC_NCI60_2000_1000_features.txt", "export_data": "/autofs/nccs-svm1_proj/med106/ncollier/repos/Supervisor/workflows/cp1/cache/GDSC_NCI60_2000_1000_test.h5", "no_feature_source": true, "no_response_source": true, "cp": true}

Create upf for inferencing runs, using inputs.txt as produced by training upf


In [208]:
from os import path
# python uno_infer.py --data CTRP_CCLE_2000_1000_test.h5 --model_file model.h5 
inputs = '/home/nick/Documents/results/cp1/inputs.txt'


model_class_ids = {}
next_id = 0

infer_upf = '/home/nick/Documents/repos/Supervisor/workflows/cp1/data/infer_upf_all.txt'
with open(infer_upf, 'w') as f_out:
    studies = ['CCLE', 'CTRP', 'gCSI', 'GDSC', 'NCI60']
    with open(inputs) as f_in:
        reader = csv.reader(f_in, delimiter="|")
        for r in reader:
            params = json.loads(r[2])
            save_path = params['save_path']
            if 'cell_feature_subset_path' in params:
                fsp = path.basename(params['cell_feature_subset_path'])
                fsp_prefix = fsp[:fsp.find('_features')]
                test_data = '{}_test.h5'.format(fsp_prefix)
                save_path = params['save_path']
                train_source = params['train_sources']
                #if fsp.find('_NCI60_') != -1:
                f_out.write('{},{},{}\n'.format(test_data, save_path, fsp_prefix))
            else:
                train_source = params['train_sources']
                for s in studies:
                    test_data = '{}.h5'.format(s)
                    f_out.write('{},{},{}_{}\n'.format(test_data, save_path, train_source, s))

In [185]:
f = "/home/nick/Documents/repos/Supervisor/workflows/cp1/scripts/counts_by_hpo.csv"
hp = {}

class Entry:

    def __init__(self, start):
        self.start = start
        self.end = -1
    
    def __repr__(self):
        return "[{}, {}]".format(self.start, self.end)

with open(f) as f_in:
    reader = csv.reader(f_in)
    next(reader)
    for row in reader:
        if row[1] == "1":
            hpo_id = int(row[3])
            h = float(row[2])
            if not hpo_id in hp:
                hp[hpo_id] = [Entry(h)]
            else:
                entry = hp[hpo_id][-1]
                if entry.end != -1:
                    hp[hpo_id].append(Entry(h))
        elif row[1] == "0":
            hpo_id = int(row[3])
            h = float(row[2])
            hp[hpo_id][-1].end = h


with open('/home/nick/Documents/repos/Supervisor/workflows/cp1/scripts/start_end.csv', 'w') as f_out:
    for k in hp:
        for e in hp[k]:
            f_out.write('{},{},{}\n'.format(e.start, e.end, k))

In [6]:
import csv
import os

f = "/home/nick/Documents/results/cp1/inference_log.txt"
prefix = '/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/'
with open(f) as f_in:
    reader = csv.reader(f_in, delimiter='|')
    for i, row in enumerate(reader):
        #train_path = row[2]
        run_id = int(os.path.basename(os.path.dirname(row[2])))
        if run_id < 200:
            print('{}{}'.format(prefix, i))


/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/0
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/1
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/2
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/3
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/4
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/5
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/6
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/7
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/8
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/9
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/10
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/11
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/12
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/13
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/14
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/15
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/16
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/17
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/18
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/19
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/20
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/21
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/22
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/23
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/24
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/25
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/26
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/27
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/28
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/29
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/30
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/31
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/32
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/33
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/34
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/35
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/36
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/37
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/38
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/39
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/40
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/41
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/42
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/43
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/44
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/45
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/46
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/47
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/48
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/49
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/50
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/51
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/52
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/53
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/54
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/55
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/56
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/57
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/58
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/59
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/60
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/61
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/62
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/63
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/64
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/65
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/66
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/67
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/68
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/69
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/70
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/71
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/72
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/73
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/74
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/75
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/76
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/77
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/78
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/79
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/80
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/81
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/82
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/83
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/84
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/85
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/86
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/87
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/88
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/89
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/90
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/91
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/92
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/93
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/94
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/95
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/96
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/97
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/98
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/99
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/100
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/101
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/102
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/103
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/104
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/105
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/106
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/107
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/108
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/109
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/110
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/111
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/112
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/113
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/114
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/115
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/116
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/117
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/118
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/119
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/120
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/121
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/122
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/123
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/124
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/125
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/126
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/127
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/128
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/129
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/130
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/131
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/132
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/133
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/134
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/135
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/136
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/137
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/138
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/139
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/140
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/141
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/142
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/143
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/144
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/145
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/146
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/147
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/148
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/149
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/150
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/151
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/152
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/153
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/154
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/155
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/156
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/157
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/158
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/159
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/160
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/161
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/162
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/163
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/164
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/165
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/166
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/167
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/168
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/169
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/170
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/171
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/172
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/173
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/174
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/175
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/176
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/177
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/178
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/179
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/180
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/181
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/182
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/183
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/184
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/185
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/186
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/187
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/188
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/189
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/190
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/191
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/192
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/193
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/194
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/195
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/196
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/197
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/198
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/199
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/200
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/201
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/202
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/203
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/204
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/205
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/206
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/207
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/208
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/209
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/210
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/211
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/212
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/213
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/214
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/215
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/216
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/217
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/218
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/219
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/220
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/221
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/222
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/223
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/224
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/225
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/226
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/227
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/228
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/229
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/230
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/231
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/232
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/233
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/234
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/235
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/236
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/237
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/238
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/239
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/240
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/241
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/242
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/243
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/244
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/245
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/246
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/247
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/248
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/249
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/250
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/251
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/252
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/253
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/254
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/255
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/256
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/257
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/258
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/259
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/260
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/261
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/262
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/263
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/264
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/265
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/266
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/267
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/268
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/269
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/270
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/271
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/272
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/273
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/274
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/275
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/276
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/277
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/278
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/279
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/280
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/281
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/282
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/283
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/284
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/285
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/286
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/287
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/288
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/289
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/290
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/291
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/292
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/293
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/294
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/295
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/296
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/297
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/298
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/299
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/300
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/301
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/302
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/303
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/304
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/305
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/306
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/307
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/308
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/309
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/310
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/311
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/312
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/313
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/314
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/315
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/316
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/317
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/318
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/319
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/320
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/321
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/322
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/323
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/324
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/325
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/326
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/327
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/328
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/329
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/330
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/331
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/332
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/333
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/334
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/335
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/336
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/337
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/338
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/339
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/340
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/341
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/342
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/343
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/344
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/345
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/346
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/347
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/348
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/349
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/350
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/351
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/352
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/353
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/354
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/355
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/356
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/357
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/358
/gpfs/alpine/med106/scratch/ncollier/experiments/infer_all_4/run/359

In [ ]: