Here we are splitting out the bit of the integration test that can be run on ABIDE NYU data in prep for running some tests in parallel
The purpose of the tests run with ABIDE NYU Data are:
In [2]:
import ciftify
from ciftify.utils import run
import os
import datetime
import logging
from glob import glob
In [4]:
#working_dir = '/home/edickie/Documents/ciftify_tests/'
working_dir = '/scratch/edickie/ciftify_intergration_tests/'
src_data_dir= os.path.join(working_dir,'src_data')
#work_from = '/home/edickie/Documents/ciftify_tests/run_2017-09-08/'
#work_from = '/scratch/edickie/ciftify_intergration_tests/run_2017-09-25_py36_sc2017-10-13/'
work_from = None
fixtures_dir = '/projects/edickie/code/ciftify/tests/integration/fixtures/'
if work_from:
new_outputs = work_from
else:
new_outputs= os.path.join(working_dir,'run_{}'.format(datetime.date.today()))
DEBUG = True
DRYRUN = True
In [5]:
logger = logging.getLogger('ciftify')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
# Get settings, and add an extra handler for the subject log
fh = logging.FileHandler(os.path.join(working_dir, 'ciftify_tests.log'))
logger.addHandler(fh)
hcp_data_dir = os.path.join(new_outputs, 'hcp')
if not os.path.exists(hcp_data_dir):
run(['mkdir','-p',hcp_data_dir])
In [6]:
def download_file(web_address, local_filename):
'''download file if it does not exist'''
if not os.path.isfile(local_filename):
run(['wget', web_address, '-O', local_filename])
if not os.path.getsize(local_filename) > 0:
os.remove(local_filename)
In [7]:
def get_recon_all_outputs(hcp_data_dir, subid):
recon_all_out = folder_contents_list(os.path.join(hcp_data_dir, subid))
recon_all_out = [x.replace(subid, 'subid') for x in recon_all_out]
return(recon_all_out)
In [8]:
abide_amazon_addy = 'https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative/Outputs/freesurfer/5.1'
subid = 'NYU_0050954'
abide_freesurfer = os.path.join(src_data_dir, 'abide', 'freesurfer')
fs_subdir = os.path.join(abide_freesurfer, subid)
if not os.path.exists(fs_subdir):
run(['mkdir', '-p', fs_subdir])
for subdir in ['mri', 'surf','label','scripts']:
localdir = os.path.join(fs_subdir, subdir)
if not os.path.exists(localdir):
run(['mkdir', '-p', localdir])
for filename in ['T1.mgz', 'aparc+aseg.mgz', 'aparc.a2009s+aseg.mgz', 'wmparc.mgz', 'brain.finalsurfs.mgz']:
download_file(os.path.join(abide_amazon_addy, subid, 'mri', filename.replace('+','%20')),
os.path.join(fs_subdir, 'mri', filename))
for surface in ['pial', 'white', 'sphere.reg', 'sphere', 'curv', 'sulc', 'thickness']:
for hemi in ['lh', 'rh']:
download_file(os.path.join(abide_amazon_addy, subid, 'surf', '{}.{}'.format(hemi,surface)),
os.path.join(fs_subdir, 'surf', '{}.{}'.format(hemi,surface)))
for labelname in ['aparc', 'aparc.a2009s', 'BA']:
for hemi in ['lh', 'rh']:
download_file(os.path.join(abide_amazon_addy, subid, 'label', '{}.{}.annot'.format(hemi,labelname)),
os.path.join(fs_subdir, 'label', '{}.{}.annot'.format(hemi,labelname)))
for script in ['recon-all.done', 'build-stamp.txt']:
download_file(os.path.join(abide_amazon_addy, subid, 'scripts', script),
os.path.join(fs_subdir, 'scripts', script))
In [33]:
run(['ciftify_recon_all', '--hcp-data-dir', hcp_data_dir,
'--fs-subjects-dir', abide_freesurfer,
'NYU_0050954'])
Out[33]:
In [34]:
run(['cifti_vis_recon_all', 'snaps', '--hcp-data-dir',hcp_data_dir, 'NYU_0050954'])
run(['cifti_vis_recon_all', 'index', '--hcp-data-dir', hcp_data_dir])
Out[34]:
In [41]:
msm_hcp = '{}_msm'.format(hcp_data_dir)
run(['mkdir', msm_hcp])
run(['ciftify_recon_all', '--hcp-data-dir', msm_hcp,
'--fs-subjects-dir', abide_freesurfer,
'NYU_0050954'])
Out[41]:
In [35]:
def download_vmhc(subid):
amazon_addy = 'https://s3.amazonaws.com/fcp-indi/data/Projects/ABIDE_Initiative/Outputs/cpac/filt_global/vmhc/{}_vmhc.nii.gz'.format(subid)
sub_vmhc = os.path.join(src_vmhc, '{}_vmhc.nii.gz'.format(subid))
download_file(amazon_addy, sub_vmhc)
src_vmhc = os.path.join(src_data_dir, 'abide', 'vmhc')
if not os.path.exists(src_vmhc):
run(['mkdir', src_vmhc])
subjects=['NYU_0050954','NYU_0050955']
for subid in subjects:
download_vmhc(subid)
In [36]:
subject = 'NYU_0050954'
run(['cifti_vis_map', 'nifti-snaps',
'--hcp-data-dir', hcp_data_dir,
'--qcdir', os.path.join(hcp_data_dir, 'abide_vmhc_vis'),
'--resample-nifti', '--colour-palette', 'fidl',
os.path.join(src_vmhc, '{}_vmhc.nii.gz'.format(subject)), subject, '{}_vmhc'.format(subject)])
Out[36]:
In [37]:
subject = 'NYU_0050954'
run(['cifti_vis_map', 'nifti-snaps',
'--hcp-data-dir', hcp_data_dir,
'--qcdir', os.path.join(hcp_data_dir, 'abide_vmhc_vis'),
'--resample-nifti',
os.path.join(src_vmhc, '{}_vmhc.nii.gz'.format(subject)), subject, '{}_vmhc_dcol'.format(subject)])
Out[37]:
In [38]:
subject = 'NYU_0050954'
run(['cifti_vis_map', 'nifti-snaps',
'--qcdir', os.path.join(hcp_data_dir, 'abide_vmhc_vis'),
'--resample-nifti', '--colour-palette', 'fidl',
os.path.join(src_vmhc, '{}_vmhc.nii.gz'.format(subject)), 'HCP_S1200_GroupAvg', '{}_vmhc'.format(subject)])
Out[38]:
In [39]:
subject = 'NYU_0050955'
run(['cifti_vis_map', 'nifti-snaps',
'--qcdir', os.path.join(hcp_data_dir, 'abide_vmhc_vis'),
'--resample-nifti',
os.path.join(src_vmhc, '{}_vmhc.nii.gz'.format(subject)), 'HCP_S1200_GroupAvg', '{}_vmhc'.format(subject)])
Out[39]:
In [40]:
run(['cifti_vis_map', 'index',
'--hcp-data-dir', '/tmp',
'--qcdir', os.path.join(hcp_data_dir, 'abide_vmhc_vis')])
Out[40]: