In [1]:

    

import ciftify
from ciftify.utils import run
import os
import datetime
import logging
from glob import glob


    

In [2]:

    

#working_dir = '/home/edickie/Documents/ciftify_tests/'
working_dir = '/scratch/edickie/ciftify_intergration_tests/'
src_data_dir= os.path.join(working_dir,'src_data')
#work_from = '/home/edickie/Documents/ciftify_tests/run_2017-09-08/'
work_from = '/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/'
#work_from = None

fixtures_dir = '/projects/edickie/code/ciftify/tests/integration/fixtures/'

if work_from:
    new_outputs = work_from
else:
    new_outputs= os.path.join(working_dir,'run_{}'.format(datetime.date.today()))
DEBUG = True
DRYRUN = True

## getting the data

freesurfer_webtgz = 'https://s3.amazonaws.com/openneuro/ds000030/ds000030_R1.0.4/compressed/ds000030_R1.0.4_derivatives_freesurfer_sub50004-50008.zip'
func_webtgz = 'https://s3.amazonaws.com/openneuro/ds000030/ds000030_R1.0.4/compressed/ds000030_R1.0.4_derivatives_sub50004-50008.zip'

subids = ['sub-50005','sub-50007']
#subids = ['sub-50005','sub-50006']

logger = logging.getLogger('ciftify')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

# Get settings, and add an extra handler for the subject log
fh = logging.FileHandler(os.path.join(working_dir, 'ciftify_tests.log'))
logger.addHandler(fh)

fs_subjects_dir = os.path.join(src_data_dir, 'ds000030_R1.0.4',
                               'derivatives','freesurfer')
hcp_data_dir = os.path.join(new_outputs, 'hcp')

if not os.path.exists(hcp_data_dir):
    run(['mkdir','-p',hcp_data_dir])


    

In [3]:

    

def download_file(web_address, local_filename):
    '''download file if it does not exist'''
    if not os.path.isfile(local_filename):
        run(['wget', web_address, '-O', local_filename])
    if not os.path.getsize(local_filename) > 0:
        os.remove(local_filename)


    

In [4]:

    

def get_recon_all_outputs(hcp_data_dir, subid):
    recon_all_out = folder_contents_list(os.path.join(hcp_data_dir, subid))
    recon_all_out = [x.replace(subid, 'subid') for x in recon_all_out]
    return(recon_all_out)


    

In [5]:

    

def folder_contents_list(path):
    '''returns a list of folder contents'''
    folder_contents = glob.glob(os.path.join(path, '**'), recursive = True)
    folder_contents = [x.replace('{}/'.format(path),'') for x in folder_contents ]
    folder_contents = folder_contents[1:] ## the first element is the path name
    return(folder_contents)


    

In [6]:

    

def seed_corr_default_out(func, seed):
    func = ciftify.meants.NibInput(func)
    seed = ciftify.meants.NibInput(seed)
    outputdir = os.path.dirname(func.path)
    outbase = '{}_{}'.format(func.base, seed.base)
    outputname = os.path.join(outputdir, outbase)
    if func.type == "nifti": outputname = '{}.nii.gz'.format(outputname)
    if func.type == "cifti": outputname = '{}.dscalar.nii'.format(outputname)
    return(outputname)

def run_vis_map(result_map, result_prefix, result_type):
    run(['cifti_vis_map', '{}-snaps'.format(result_type), '--hcp-data-dir', hcp_data_dir, 
     result_map, subid, result_prefix])

def run_seedcorr_peaktable(result_map):
    run(['ciftify_peaktable',  
         '--min-threshold', '-0.5', 
         '--max-threshold', '0.5', 
         '--no-cluster-dlabel', 
         result_map])

def run_seed_corr_fmri_test(func_cifti, hcp_data_dir, roi_dir):
    ''' runs one of the seed corrs and then peak table to get a csv to test for each dtseries'''
    subid = os.path.basename(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(func_cifti)))))
    atlas_vol =  os.path.join(hcp_data_dir, subid, 'MNINonLinear', 'wmparc.nii.gz')
    struct = 'RIGHT-PUTAMEN'
    putamen_vol_seed_mask = os.path.join(roi_dir, '{}_{}_vol.nii.gz'.format(subid, struct))
    if not os.path.exists(putamen_vol_seed_mask):
        run(['wb_command', '-volume-label-to-roi', atlas_vol, putamen_vol_seed_mask, '-name', struct])
    run(['ciftify_seed_corr', func_cifti, putamen_vol_seed_mask])
    run_seedcorr_peaktable(seed_corr_default_out(func_cifti, putamen_vol_seed_mask))


    

In [7]:

    

subids = ['sub-50005','sub-50007']

if not os.path.exists(src_data_dir):
    run(['mkdir','-p',src_data_dir])

for subid in subids:
    sub_fs_path = os.path.join('ds000030_R1.0.4','derivatives','freesurfer', subid)
if not os.path.exists(os.path.join(src_data_dir, sub_fs_path)):
    if not os.path.exists(os.path.join(src_data_dir, 
                                       os.path.basename(freesurfer_webtgz))):        
        run(['wget', '-P', src_data_dir, freesurfer_webtgz])

        run(['unzip', 
             os.path.join(src_data_dir, 
                          os.path.basename(freesurfer_webtgz)), 
             os.path.join(sub_fs_path,'*'),
             '-d', src_data_dir])

for subid in subids:
    sub_file = os.path.join('ds000030_R1.0.4','derivatives',
                            'fmriprep', subid,'func',
                            '{}_task-rest_bold_space-T1w_preproc.nii.gz'.format(subid))
    if not os.path.exists(os.path.join(src_data_dir,sub_file)):
        if not os.path.exists(os.path.join(src_data_dir, 
                                           os.path.basename(func_webtgz))):
            run(['wget', '-P', src_data_dir, func_webtgz])

            run(['unzip', 
                 os.path.join(src_data_dir, 
                              os.path.basename(func_webtgz)), 
                 sub_file,
                 '-d', src_data_dir])

for subid in subids:
    sub_file = os.path.join('ds000030_R1.0.4','derivatives',
                            'fmriprep', subid,'func',
                            '{}_task-rest_bold_space-MNI152NLin2009cAsym_preproc.nii.gz'.format(subid))
    if not os.path.exists(os.path.join(src_data_dir,sub_file)):
        if not os.path.exists(os.path.join(src_data_dir, 
                                           os.path.basename(func_webtgz))):
            run(['wget', '-P', src_data_dir, func_webtgz])

            run(['unzip', 
                 os.path.join(src_data_dir, 
                              os.path.basename(func_webtgz)), 
                 sub_file,
                 '-d', src_data_dir])


    

In [ ]:

    

run(['ciftify_recon_all', '--resample-to-T1w32k', '--surf-reg', 'MSMSulc', '--ciftify-work-dir', hcp_data_dir,
    '--fs-subjects-dir', fs_subjects_dir, 
    subids[0]])


    

In [ ]:

    

run(['ciftify_recon_all', '--surf-reg', 'FS', '--hcp-data-dir', hcp_data_dir,
    '--fs-subjects-dir', fs_subjects_dir, 
    subids[1]])


    

In [ ]:

    

run(['cifti_vis_recon_all', 'snaps', '--hcp-data-dir',hcp_data_dir, 'NYU_0050954'])
run(['cifti_vis_recon_all', 'index', '--hcp-data-dir', hcp_data_dir])


    

In [41]:

    

subid=subids[0]
native_func = os.path.join(src_data_dir,'ds000030_R1.0.4','derivatives',
                 'fmriprep', subid,'func',
                 '{}_task-rest_bold_space-native_preproc.nii.gz'.format(subid))
mat_file = os.path.join(src_data_dir, 'ds000030_R1.0.4','derivatives',
                            'fmriprep', subid,'func',
                            '{}_task-rest_bold_T1_to_EPI.mat'.format(subid))

with open(mat_file, "w") as text_file:
    text_file.write('''1.03  -0.015  0.0025  -15.0
0.014  1.01  -0.005  -11.9
-0.007 0.01  0.99  2
0  0  0  1
''')

t1_func = os.path.join(src_data_dir,'ds000030_R1.0.4','derivatives',
                 'fmriprep', subid,'func',
                 '{}_task-rest_bold_space-T1w_preproc.nii.gz'.format(subid))
if not os.path.exists(native_func):
    run(['flirt', '-in', t1_func, '-ref', t1_func, 
         '-out', native_func, '-init', mat_file, '-applyxfm'])

run(['ciftify_subject_fmri','--SmoothingFWHM', '12', '--FLIRT-to-T1w',
     '--OutputSurfDiagnostics',
    '--hcp-data-dir', hcp_data_dir, 
    native_func, subid, 'rest_test1'])


    

    




Running: ciftify_subject_fmri --SmoothingFWHM 12 --FLIRT-to-T1w --OutputSurfDiagnostics --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp /scratch/edickie/ciftify_intergration_tests/src_data/ds000030_R1.0.4/derivatives/fmriprep/sub-50005/func/sub-50005_task-rest_bold_space-native_preproc.nii.gz sub-50005 rest_test1
{'--DilateBelowPct': None,
 '--FLIRT-to-T1w': True,
 '--OutputSurfDiagnostics': True,
 '--SmoothingFWHM': '12',
 '--already-in-MNI': False,
 '--ciftify-conf': None,
 '--ciftify-work-dir': None,
 '--debug': False,
 '--dry-run': False,
 '--func-ref': 'first_vol',
 '--hcp-data-dir': '/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp',
 '--help': False,
 '--surf-reg': 'FS',
 '--verbose': False,
 '<func.nii.gz>': '/scratch/edickie/ciftify_intergration_tests/src_data/ds000030_R1.0.4/derivatives/fmriprep/sub-50005/func/sub-50005_task-rest_bold_space-native_preproc.nii.gz',
 '<subject>': 'sub-50005',
 '<task_label>': 'rest_test1'}

Argument --hcp-data-dir has been deprecated. Please instead use --ciftify-work-dir in the future.
Smoothing kernels greater than 6mm FWHM are not recommended by the HCP, 12.0 specified
/mnt/tigrlab/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp/sub-50005/MNINonLinear/Results/rest_test1/RibbonVolumeToSurfaceMapping already exists







    
Out[41]:





0

In [53]:

    

run(['cifti_vis_fmri', 'snaps', '--SmoothingFWHM', '12', '--hcp-data-dir',hcp_data_dir, 'rest_test1', subid])


    

    




Running: cifti_vis_fmri snaps --SmoothingFWHM 12 --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp rest_test1 sub-50005
cmd: cifti_vis_fmri snaps --SmoothingFWHM 12 --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp rest_test1 sub-50005 
 Failed with returncode 1
[cifti_vis_fmri] WARNING: The 'snaps' argument has be deprecated. Please use 'subject' in the future.
[ciftify.utils] WARNING: Argument --hcp-data-dir has been deprecated. Please instead use --ciftify-work-dir in the future.
[cifti_vis_fmri] ERROR: Expected smoothed fmri file /mnt/tigrlab/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp/sub-50005/MNINonLinear/Results/rest_test1/rest_test1_Atlas_s12.dtseries.nii not found.To generate temporary smoothed file for visulizations use the --smooth-con flag instead







    
Out[53]:





1

In [52]:

    

run(['ciftify_subject_fmri', '--SmoothingFWHM', '4', 
    '--hcp-data-dir', hcp_data_dir, '--OutputSurfDiagnostics',
    '--surf-reg', 'MSMSulc', t1_func,
    subid, 'rest_t1first_MSMSulc'])


    

    




Running: ciftify_subject_fmri --SmoothingFWHM 4 --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp --OutputSurfDiagnostics --surf-reg MSMSulc /scratch/edickie/ciftify_intergration_tests/src_data/ds000030_R1.0.4/derivatives/fmriprep/sub-50005/func/sub-50005_task-rest_bold_space-T1w_preproc.nii.gz sub-50005 rest_t1first_MSMSulc
{'--DilateBelowPct': None,
 '--FLIRT-to-T1w': False,
 '--OutputSurfDiagnostics': True,
 '--SmoothingFWHM': '4',
 '--already-in-MNI': False,
 '--ciftify-conf': None,
 '--ciftify-work-dir': None,
 '--debug': False,
 '--dry-run': False,
 '--func-ref': 'first_vol',
 '--hcp-data-dir': '/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp',
 '--help': False,
 '--surf-reg': 'MSMSulc',
 '--verbose': False,
 '<func.nii.gz>': '/scratch/edickie/ciftify_intergration_tests/src_data/ds000030_R1.0.4/derivatives/fmriprep/sub-50005/func/sub-50005_task-rest_bold_space-T1w_preproc.nii.gz',
 '<subject>': 'sub-50005',
 '<task_label>': 'rest_t1first_MSMSulc'}

Argument --hcp-data-dir has been deprecated. Please instead use --ciftify-work-dir in the future.
/mnt/tigrlab/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp/sub-50005/MNINonLinear/Results/rest_t1first_MSMSulc/RibbonVolumeToSurfaceMapping already exists







    
Out[52]:





0

In [54]:

    

run(['cifti_vis_fmri', 'snaps', '--SmoothingFWHM', '4', '--hcp-data-dir',hcp_data_dir, 'rest_t1first_MSMSulc', subid])


    

    




Running: cifti_vis_fmri snaps --SmoothingFWHM 4 --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp rest_t1first_MSMSulc sub-50005
cmd: cifti_vis_fmri snaps --SmoothingFWHM 4 --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp rest_t1first_MSMSulc sub-50005 
 Failed with returncode 1
[cifti_vis_fmri] WARNING: The 'snaps' argument has be deprecated. Please use 'subject' in the future.
[ciftify.utils] WARNING: Argument --hcp-data-dir has been deprecated. Please instead use --ciftify-work-dir in the future.
[cifti_vis_fmri] ERROR: Expected smoothed fmri file /mnt/tigrlab/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp/sub-50005/MNINonLinear/Results/rest_t1first_MSMSulc/rest_t1first_MSMSulc_Atlas_s4.dtseries.nii not found.To generate temporary smoothed file for visulizations use the --smooth-con flag instead







    
Out[54]:





1

In [48]:

    

subid = subids[0]
run(['ciftify_subject_fmri', '--already-in-MNI', '--SmoothingFWHM', '6', 
    '--hcp-data-dir', hcp_data_dir, '--OutputSurfDiagnostics',
    os.path.join(src_data_dir,'ds000030_R1.0.4','derivatives',
                 'fmriprep', subid,'func',
                 '{}_task-rest_bold_space-MNI152NLin2009cAsym_preproc.nii.gz'.format(subid)),
    subid, 'rest_bad_transform'])


    

    




Running: ciftify_subject_fmri --already-in-MNI --SmoothingFWHM 6 --hcp-data-dir /scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp --OutputSurfDiagnostics /scratch/edickie/ciftify_intergration_tests/src_data/ds000030_R1.0.4/derivatives/fmriprep/sub-50005/func/sub-50005_task-rest_bold_space-MNI152NLin2009cAsym_preproc.nii.gz sub-50005 rest_bad_transform
{'--DilateBelowPct': None,
 '--FLIRT-to-T1w': False,
 '--OutputSurfDiagnostics': True,
 '--SmoothingFWHM': '6',
 '--already-in-MNI': True,
 '--ciftify-conf': None,
 '--ciftify-work-dir': None,
 '--debug': False,
 '--dry-run': False,
 '--func-ref': 'first_vol',
 '--hcp-data-dir': '/scratch/edickie/ciftify_intergration_tests/run_2018-04-16/hcp',
 '--help': False,
 '--surf-reg': 'FS',
 '--verbose': False,
 '<func.nii.gz>': '/scratch/edickie/ciftify_intergration_tests/src_data/ds000030_R1.0.4/derivatives/fmriprep/sub-50005/func/sub-50005_task-rest_bold_space-MNI152NLin2009cAsym_preproc.nii.gz',
 '<subject>': 'sub-50005',
 '<task_label>': 'rest_bad_transform'}

Argument --hcp-data-dir has been deprecated. Please instead use --ciftify-work-dir in the future.







    
Out[48]:





0