Goal

Accuracy as a function of isotope incorporation & community similarity among replicate microcosms

Variable parameters:

  • shared % of taxa
    • 80, 85, 90, 95, 100
  • permuted % of rank abundances
    • 20, 15, 10, 5, 0
  • atom % isotope incorporation
    • 100
  • % taxa that incorporate
    • 10
  • n-reps (stocastic: taxon abundances & which incorporate)
    • 10

Setting paths


In [5]:
import os

# paths
workDir = '/home/nick/notebook/SIPSim/dev/bac_genome1147/'
buildDir = os.path.join(workDir, 'microBetaDiv')
R_dir = '/home/nick/notebook/SIPSim/lib/R/'

fragFile = '/home/nick/notebook/SIPSim/dev/bac_genome1147/validation/ampFrags_kde.pkl'
genome_index = '/var/seq_data/ncbi_db/genome/Jan2016/bac_complete_spec-rep1_rn/genome_index.txt'

Init


In [6]:
import glob
import itertools
import nestly

In [7]:
%load_ext rpy2.ipython
%load_ext pushnote

In [8]:
%%R
library(ggplot2)
library(dplyr)
library(tidyr)
library(gridExtra)


/opt/anaconda/lib/python2.7/site-packages/rpy2/robjects/functions.py:106: UserWarning: 
Attaching package: ‘dplyr’


  res = super(Function, self).__call__(*new_args, **new_kwargs)
/opt/anaconda/lib/python2.7/site-packages/rpy2/robjects/functions.py:106: UserWarning: The following objects are masked from ‘package:stats’:

    filter, lag


  res = super(Function, self).__call__(*new_args, **new_kwargs)
/opt/anaconda/lib/python2.7/site-packages/rpy2/robjects/functions.py:106: UserWarning: The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union


  res = super(Function, self).__call__(*new_args, **new_kwargs)
/opt/anaconda/lib/python2.7/site-packages/rpy2/robjects/functions.py:106: UserWarning: 
Attaching package: ‘gridExtra’


  res = super(Function, self).__call__(*new_args, **new_kwargs)
/opt/anaconda/lib/python2.7/site-packages/rpy2/robjects/functions.py:106: UserWarning: The following object is masked from ‘package:dplyr’:

    combine


  res = super(Function, self).__call__(*new_args, **new_kwargs)

In [9]:
if not os.path.isdir(buildDir):
    os.makedirs(buildDir)
%cd $buildDir


/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv

BD min/max


In [10]:
## min G+C cutoff
min_GC = 13.5
## max G+C cutoff
max_GC = 80
## max G+C shift
max_13C_shift_in_BD = 0.036


min_BD = min_GC/100.0 * 0.098 + 1.66    
max_BD = max_GC/100.0 * 0.098 + 1.66    

max_BD = max_BD + max_13C_shift_in_BD

print 'Min BD: {}'.format(min_BD)
print 'Max BD: {}'.format(max_BD)


Min BD: 1.67323
Max BD: 1.7744

Nestly

  • assuming fragments already simulated

In [11]:
# making an experimental design file for qSIP
x = range(1,7)
y = ['control', 'treatment']

expDesignFile = os.path.join(buildDir, 'qSIP_exp_design.txt')
with open(expDesignFile, 'wb') as outFH:
    for i,z in itertools.izip(x,itertools.cycle(y)):
        line = '\t'.join([str(i),z])
        outFH.write(line + '\n')

!head $expDesignFile


1	control
2	treatment
3	control
4	treatment
5	control
6	treatment

Nestly params


In [108]:
# building tree structure
nest = nestly.Nest()

# varying params: test
#nest.add('shared_perc', [90, 95])
#nest.add('perm_perc', [5, 10])
#nest.add('rep', range(1,4))

# varying params
nest.add('shared_perc', [80, 85, 90, 95, 100])
nest.add('perm_perc', [0, 5, 10, 15, 20])
nest.add('rep', range(1,11))

## set params
nest.add('percIncorp', [100], create_dir=False)
nest.add('percTaxa', [10], create_dir=False)

nest.add('abs', ['1e9'], create_dir=False)
nest.add('np', [10], create_dir=False)
nest.add('Monte_rep', [100000], create_dir=False)
nest.add('subsample_dist', ['lognormal'], create_dir=False)
nest.add('subsample_mean', [9.432], create_dir=False)
nest.add('subsample_scale', [0.5], create_dir=False)
nest.add('subsample_min', [10000], create_dir=False)
nest.add('subsample_max', [30000], create_dir=False)
nest.add('min_BD', [min_BD], create_dir=False)
nest.add('max_BD', [max_BD], create_dir=False)
nest.add('DBL_scaling', [0.5], create_dir=False)
nest.add('bandwidth', [0.8], create_dir=False)
nest.add('heavy_BD_min', [1.71], create_dir=False)
nest.add('heavy_BD_max', [1.75], create_dir=False)
nest.add('topTaxaToPlot', [100], create_dir=False)
nest.add('padj', [0.1], create_dir=False)
nest.add('log2', [0.25], create_dir=False)

### input/output files
nest.add('buildDir', [buildDir], create_dir=False)
nest.add('R_dir', [R_dir], create_dir=False)
nest.add('genome_index', [genome_index], create_dir=False)
nest.add('fragFile', [fragFile], create_dir=False)
nest.add('exp_design', [expDesignFile], create_dir=False)


# building directory tree
nest.build(buildDir)

# bash file to run
bashFile = os.path.join(buildDir, 'SIPSimRun.sh')

Experimental design


In [9]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_expDesign.sh'
bashFileTmp


Out[9]:
'/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_expDesign.sh'

In [10]:
%%writefile $bashFileTmp
#!/bin/bash

echo '#-- Experimental design --#'

echo '# Making an isotope incorporation config file'
echo '## 3 replicate gradients for control & treatment'
SIPSim incorpConfigExample \
  --percIncorpUnif {percIncorp} \
  --n_reps 3 \
  > incorp.config

echo '# Selecting incorporator taxa'
echo '## This is to make the gradient replicates consistent (qSIP finds mean among replicates)'
SIPSim KDE_selectTaxa \
    -p {percTaxa} \
    {fragFile} \
    > incorporators.txt

echo '# Creating a community file (3 replicate control, 3 replicate treatment)'
SIPSim communities \
    --richness 950 \
    --config incorp.config \
    --shared_perc {shared_perc} \
    --perm_perc {perm_perc} \
    {genome_index} \
    > comm.txt    

echo '# simulating gradient fractions'
SIPSim gradient_fractions \
    --BD_min {min_BD} \
    --BD_max {max_BD} \
    comm.txt \
    > fracs.txt


Writing /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_expDesign.sh

In [11]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file exp_design.log -j 10


2016-04-20 11:39:18,435 * INFO * Template: ./SIPSimRun_expDesign.sh
2016-04-20 11:39:18,437 * INFO * [181437] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7
2016-04-20 11:39:18,439 * INFO * [181438] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9
2016-04-20 11:39:18,441 * INFO * [181439] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8
2016-04-20 11:39:18,442 * INFO * [181442] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3
2016-04-20 11:39:18,444 * INFO * [181444] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10
2016-04-20 11:39:18,446 * INFO * [181446] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2
2016-04-20 11:39:18,447 * INFO * [181448] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1
2016-04-20 11:39:18,448 * INFO * [181450] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4
2016-04-20 11:39:18,450 * INFO * [181452] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5
2016-04-20 11:39:18,451 * INFO * [181454] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6
2016-04-20 11:39:23,797 * INFO * [181439] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8 Finished with 0
2016-04-20 11:39:23,798 * INFO * [182537] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7
2016-04-20 11:39:23,823 * INFO * [181444] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10 Finished with 0
2016-04-20 11:39:23,825 * INFO * [182550] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9
2016-04-20 11:39:23,964 * INFO * [181442] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3 Finished with 0
2016-04-20 11:39:23,965 * INFO * [182565] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8
2016-04-20 11:39:23,973 * INFO * [181438] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9 Finished with 0
2016-04-20 11:39:23,974 * INFO * [182567] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3
2016-04-20 11:39:24,018 * INFO * [181437] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7 Finished with 0
2016-04-20 11:39:24,020 * INFO * [182571] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10
2016-04-20 11:39:24,021 * INFO * [181454] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6 Finished with 0
2016-04-20 11:39:24,022 * INFO * [182573] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2
2016-04-20 11:39:24,066 * INFO * [181448] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1 Finished with 0
2016-04-20 11:39:24,068 * INFO * [182586] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1
2016-04-20 11:39:24,169 * INFO * [181450] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4 Finished with 0
2016-04-20 11:39:24,171 * INFO * [182612] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4
2016-04-20 11:39:24,233 * INFO * [181452] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5 Finished with 0
2016-04-20 11:39:24,235 * INFO * [182618] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5
2016-04-20 11:39:24,385 * INFO * [181446] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2 Finished with 0
2016-04-20 11:39:24,387 * INFO * [182668] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6
2016-04-20 11:39:29,112 * INFO * [182537] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7 Finished with 0
2016-04-20 11:39:29,114 * INFO * [183615] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7
2016-04-20 11:39:29,370 * INFO * [182550] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9 Finished with 0
2016-04-20 11:39:29,371 * INFO * [183684] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9
2016-04-20 11:39:29,396 * INFO * [182565] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8 Finished with 0
2016-04-20 11:39:29,397 * INFO * [183689] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8
2016-04-20 11:39:29,418 * INFO * [182567] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3 Finished with 0
2016-04-20 11:39:29,419 * INFO * [183691] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3
2016-04-20 11:39:29,598 * INFO * [182612] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4 Finished with 0
2016-04-20 11:39:29,600 * INFO * [183723] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/10
2016-04-20 11:39:29,624 * INFO * [182573] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2 Finished with 0
2016-04-20 11:39:29,626 * INFO * [183725] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/2
2016-04-20 11:39:29,709 * INFO * [182586] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1 Finished with 0
2016-04-20 11:39:29,711 * INFO * [183751] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/1
2016-04-20 11:39:29,743 * INFO * [182618] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5 Finished with 0
2016-04-20 11:39:29,744 * INFO * [183774] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/4
2016-04-20 11:39:29,809 * INFO * [182571] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10 Finished with 0
2016-04-20 11:39:29,811 * INFO * [183780] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/5
2016-04-20 11:39:30,043 * INFO * [182668] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6 Finished with 0
2016-04-20 11:39:30,044 * INFO * [183835] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/6
2016-04-20 11:39:34,543 * INFO * [183615] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7 Finished with 0
2016-04-20 11:39:34,544 * INFO * [184732] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/7
2016-04-20 11:39:34,902 * INFO * [183691] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3 Finished with 0
2016-04-20 11:39:34,903 * INFO * [184846] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/9
2016-04-20 11:39:34,939 * INFO * [183684] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9 Finished with 0
2016-04-20 11:39:34,941 * INFO * [184850] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/8
2016-04-20 11:39:34,941 * INFO * [183689] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8 Finished with 0
2016-04-20 11:39:34,948 * INFO * [184852] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/3
2016-04-20 11:39:35,137 * INFO * [183751] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/1 Finished with 0
2016-04-20 11:39:35,143 * INFO * [184872] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/10
2016-04-20 11:39:35,144 * INFO * [183725] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/2 Finished with 0
2016-04-20 11:39:35,145 * INFO * [184873] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/2
2016-04-20 11:39:35,193 * INFO * [183723] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/10 Finished with 0
2016-04-20 11:39:35,195 * INFO * [184876] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/1
2016-04-20 11:39:35,202 * INFO * [183774] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/4 Finished with 0
2016-04-20 11:39:35,204 * INFO * [184878] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/4
2016-04-20 11:39:35,239 * INFO * [183780] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/5 Finished with 0
2016-04-20 11:39:35,248 * INFO * [184883] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/5
2016-04-20 11:39:35,475 * INFO * [183835] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/6 Finished with 0
2016-04-20 11:39:35,476 * INFO * [184959] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/6
2016-04-20 11:39:39,846 * INFO * [184732] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/7 Finished with 0
2016-04-20 11:39:39,848 * INFO * [185841] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/5/7
2016-04-20 11:39:40,306 * INFO * [184846] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/9 Finished with 0
2016-04-20 11:39:40,308 * INFO * [185950] Started ./SIPSimRun_expDesign.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/5/9
**OUTPUT MUTED**

In [12]:
%pushnote exp_design complete: $buildDir

SIPSim pipeline


In [13]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_SIPSim-pipeline.sh'
bashFileTmp


Out[13]:
'/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_SIPSim-pipeline.sh'

In [ ]:
%%writefile $bashFileTmp
#!/bin/bash

echo '#-- SIPSim pipeline --#'    
    
echo '# Adding diffusion'    
SIPSim diffusion \
    -n {Monte_rep} \
    --bw {bandwidth} \
    --np {np} \
    {fragFile} \
    > ampFrags_KDE_dif.pkl    

echo '# Adding DBL contamination; abundance-weighted smearing'
SIPSim DBL \
    -n {Monte_rep} \
    --comm comm.txt \
    --commx {DBL_scaling} \
    --np {np} \
    ampFrags_KDE_dif.pkl \
    > ampFrags_KDE_dif_DBL.pkl 

echo '# Adding isotope incorporation to BD distribution'
SIPSim isotope_incorp \
    -n {Monte_rep} \
    --comm comm.txt \
    --taxa incorporators.txt \
    --np {np} \
    ampFrags_KDE_dif_DBL.pkl \
    incorp.config \
    > ampFrags_KDE_dif_DBL_inc.pkl

echo '# Simulating an OTU table'
SIPSim OTU_table \
    --abs {abs} \
    --np {np} \
    ampFrags_KDE_dif_DBL_inc.pkl \
    comm.txt \
    fracs.txt \
    > OTU_abs{abs}.txt
    
echo '# Simulating PCR'
SIPSim OTU_PCR \
    OTU_abs{abs}.txt \
    > OTU_abs{abs}_PCR.txt    
    
echo '# Subsampling from the OTU table (simulating sequencing of the DNA pool)'
SIPSim OTU_subsample \
    --dist {subsample_dist} \
    --dist_params mean:{subsample_mean},sigma:{subsample_scale} \
    --min_size {subsample_min} \
    --max_size {subsample_max} \
    OTU_abs{abs}_PCR.txt \
    > OTU_abs{abs}_PCR_sub.txt
        
echo '# Making a wide-formatted table'
SIPSim OTU_wideLong -w \
    OTU_abs{abs}_PCR_sub.txt \
    > OTU_abs{abs}_PCR_sub_w.txt
    
echo '# Making metadata (phyloseq: sample_data)'
SIPSim OTU_sampleData \
    OTU_abs{abs}_PCR_sub.txt \
    > OTU_abs{abs}_PCR_sub_meta.txt
       

#-- removing large intermediate files --#
rm -f ampFrags_KDE_dif.pkl
rm -f ampFrags_KDE_dif_DBL.pkl
rm -f ampFrags_KDE_dif_DBL_inc.pkl


Writing /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_SIPSim-pipeline.sh

In [ ]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file SIPSim_pipeline.log -j 2


2016-04-20 11:41:36,793 * INFO * Template: ./SIPSimRun_SIPSim-pipeline.sh
2016-04-20 11:41:36,795 * INFO * [210119] Started ./SIPSimRun_SIPSim-pipeline.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7
2016-04-20 11:41:36,797 * INFO * [210120] Started ./SIPSimRun_SIPSim-pipeline.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9

In [ ]:
%pushnote SIPSim pipeline complete: $buildDir

Summary of simulated data


In [ ]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_SIPSim-summary.sh'
bashFileTmp

In [ ]:
%%writefile $bashFileTmp
#!/bin/bash
   
# plotting 'raw' taxon abundances
SIPSimR OTU_taxonAbund \
    OTU_abs{abs}.txt \
    -r {topTaxaToPlot} \
    -o OTU_abs{abs}

# plotting 'sequenced' taxon abundances
SIPSimR OTU_taxonAbund \
    OTU_abs{abs}_PCR_sub.txt \
    -r {topTaxaToPlot} \
    -o OTU_abs{abs}_PCR_sub

In [ ]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file SIPSim_summary.log -j 10

HR-SIP: DESeq2


In [ ]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_HRSIP.sh'
bashFileTmp

In [ ]:
%%writefile $bashFileTmp
#!/bin/bash

# phyloseq
## making phyloseq object from OTU table
SIPSimR phyloseq_make \
    OTU_abs{abs}_PCR_sub_w.txt \
    -s OTU_abs{abs}_PCR_sub_meta.txt \
    > OTU_abs{abs}_PCR_sub.physeq

## filtering phyloseq object to just 'heavy' fractions
SIPSimR phyloseq_edit \
    OTU_abs{abs}_PCR_sub.physeq \
    --BD_min {heavy_BD_min} \
    --BD_max {heavy_BD_max} \
    > OTU_abs{abs}_PCR_sub_filt.physeq

## making ordination
SIPSimR phyloseq_ordination \
    OTU_abs{abs}_PCR_sub_filt.physeq \
    OTU_abs{abs}_PCR_sub_filt_bray-NMDS.pdf

# DESeq2
SIPSimR phyloseq_DESeq2 \
    --log2 {log2} \
    --hypo greater \
    --cont 1,3,5 \
    --treat 2,4,6 \
    OTU_abs{abs}_PCR_sub_filt.physeq \
    > OTU_abs{abs}_PCR_sub_filt_DESeq2

In [ ]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file HR-SIP.log -j 10

In [ ]:
%pushnote HR-SIP complete: $buildDir

MW-HR-SIP

  • BD windows: 1.70-1.73,1.72-1.75,1.74-1.77

In [133]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_MWHRSIP.sh'
bashFileTmp


Out[133]:
'/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_MWHRSIP.sh'

In [134]:
%%writefile $bashFileTmp
#!/bin/bash

## HR SIP pipeline
SIPSimR phyloseq_DESeq2 \
    --log2 {log2} \
    --hypo greater \
    --cont 1,3,5 \
    --treat 2,4,6 \
    --occur_all 0.0,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5 \
    -w 1.70-1.73,1.72-1.75,1.74-1.77 \
    --all OTU_abs1e9_PCR_sub_MW-all.txt \
    OTU_abs{abs}_PCR_sub.physeq \
    > OTU_abs{abs}_PCR_sub_filt_MW_DESeq2


Writing /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_MWHRSIP.sh

In [135]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file MW-HR-SIP.log -j 14


2016-05-15 08:38:35,511 * INFO * Template: ./SIPSimRun_MWHRSIP.sh
2016-05-15 08:38:35,513 * INFO * [197078] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7
2016-05-15 08:38:35,515 * INFO * [197079] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9
2016-05-15 08:38:35,516 * INFO * [197081] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8
2016-05-15 08:38:35,517 * INFO * [197083] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3
2016-05-15 08:38:35,519 * INFO * [197085] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10
2016-05-15 08:38:35,522 * INFO * [197087] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2
2016-05-15 08:38:35,524 * INFO * [197090] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1
2016-05-15 08:38:35,525 * INFO * [197091] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4
2016-05-15 08:38:35,527 * INFO * [197093] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5
2016-05-15 08:38:35,529 * INFO * [197095] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6
2016-05-15 08:38:35,536 * INFO * [197097] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/7
2016-05-15 08:38:35,538 * INFO * [197099] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/9
2016-05-15 08:38:35,540 * INFO * [197101] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/8
2016-05-15 08:38:35,541 * INFO * [197103] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/3
2016-05-15 08:40:45,111 * INFO * [197079] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9 Finished with 0
2016-05-15 08:40:45,113 * INFO * [197811] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/10
2016-05-15 08:40:47,727 * INFO * [197081] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8 Finished with 0
2016-05-15 08:40:47,729 * INFO * [197854] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/2
2016-05-15 08:40:49,410 * INFO * [197078] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7 Finished with 0
2016-05-15 08:40:49,413 * INFO * [197923] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/1
2016-05-15 08:40:49,789 * INFO * [197085] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10 Finished with 0
2016-05-15 08:40:49,792 * INFO * [197935] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/4
2016-05-15 08:40:52,314 * INFO * [197091] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4 Finished with 0
2016-05-15 08:40:52,316 * INFO * [197979] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/5
2016-05-15 08:40:56,692 * INFO * [197093] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5 Finished with 0
2016-05-15 08:40:56,694 * INFO * [198042] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/6
2016-05-15 08:40:57,807 * INFO * [197095] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6 Finished with 0
2016-05-15 08:40:57,809 * INFO * [198076] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/7
2016-05-15 08:40:58,493 * INFO * [197103] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/3 Finished with 0
2016-05-15 08:40:58,496 * INFO * [198115] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/9
2016-05-15 08:41:00,379 * INFO * [197101] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/8 Finished with 0
2016-05-15 08:41:00,383 * INFO * [198154] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/8
2016-05-15 08:41:00,889 * INFO * [197087] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2 Finished with 0
2016-05-15 08:41:00,892 * INFO * [198175] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/3
2016-05-15 08:41:00,929 * INFO * [197097] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/7 Finished with 0
2016-05-15 08:41:00,931 * INFO * [198177] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/10
2016-05-15 08:41:02,287 * INFO * [197083] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3 Finished with 0
2016-05-15 08:41:02,288 * INFO * [198231] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/2
2016-05-15 08:41:02,464 * INFO * [197090] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1 Finished with 0
2016-05-15 08:41:02,466 * INFO * [198235] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/1
2016-05-15 08:41:07,102 * INFO * [197099] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/9 Finished with 0
2016-05-15 08:41:07,105 * INFO * [198303] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/4
2016-05-15 08:43:10,374 * INFO * [197923] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/1 Finished with 0
2016-05-15 08:43:10,377 * INFO * [198761] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/5
2016-05-15 08:43:12,180 * INFO * [197811] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/10 Finished with 0
2016-05-15 08:43:12,183 * INFO * [198786] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/6
2016-05-15 08:43:14,322 * INFO * [197935] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/4 Finished with 0
2016-05-15 08:43:14,324 * INFO * [198811] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7
2016-05-15 08:43:15,224 * INFO * [197854] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/2 Finished with 0
2016-05-15 08:43:15,226 * INFO * [198836] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9
2016-05-15 08:43:17,768 * INFO * [197979] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/5 Finished with 0
2016-05-15 08:43:17,770 * INFO * [198881] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8
2016-05-15 08:43:20,168 * INFO * [198042] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/6 Finished with 0
2016-05-15 08:43:20,170 * INFO * [198929] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3
2016-05-15 08:43:27,096 * INFO * [198076] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/7 Finished with 0
2016-05-15 08:43:27,099 * INFO * [198992] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10
2016-05-15 08:43:30,320 * INFO * [198175] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/3 Finished with 0
2016-05-15 08:43:30,322 * INFO * [199017] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2
2016-05-15 08:43:32,481 * INFO * [198303] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/4 Finished with 0
2016-05-15 08:43:32,483 * INFO * [199042] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1
2016-05-15 08:43:32,813 * INFO * [198177] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/10 Finished with 0
2016-05-15 08:43:32,815 * INFO * [199048] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4
2016-05-15 08:43:33,544 * INFO * [198154] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/8 Finished with 0
2016-05-15 08:43:33,546 * INFO * [199092] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5
2016-05-15 08:43:36,258 * INFO * [198235] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/1 Finished with 0
2016-05-15 08:43:36,260 * INFO * [199123] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6
2016-05-15 08:43:37,850 * INFO * [198231] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/2 Finished with 0
2016-05-15 08:43:37,855 * INFO * [199154] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7
2016-05-15 08:43:39,311 * INFO * [198115] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/9 Finished with 0
2016-05-15 08:43:39,313 * INFO * [199185] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9
2016-05-15 08:45:29,079 * INFO * [198811] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7 Finished with 0
2016-05-15 08:45:29,081 * INFO * [199668] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8
2016-05-15 08:45:30,490 * INFO * [198881] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8 Finished with 0
2016-05-15 08:45:30,492 * INFO * [199699] Started ./SIPSimRun_MWHRSIP.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3
**OUTPUT MUTED**

In [136]:
%pushnote MW-HR-SIP complete: $buildDir

MW-HR-SIP: debug

  • BD windows: 1.70-1.73,1.72-1.75,1.74-1.77
  • l2fc cutoff of 0.5 (instead of 0.25)

In [206]:
# bashFileTmp = os.path.splitext(bashFile)[0] + '_MWHRSIP2.sh'
# bashFileTmp

In [207]:
%%writefile $bashFileTmp
#!/bin/bash

# ## HR SIP pipeline
# SIPSimR phyloseq_DESeq2 \
#     --log2 0.5 \
#     --hypo greater \
#     --cont 1,3,5 \
#     --treat 2,4,6 \
#     --occur_all 0.0,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5 \
#     -w 1.70-1.73,1.72-1.75,1.74-1.77 \
#     --all OTU_abs1e9_PCR_sub_MW0.5-all.txt \
#     OTU_abs{abs}_PCR_sub.physeq \
#     > OTU_abs{abs}_PCR_sub_filt_MW0.5_DESeq2


Overwriting /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_cMtx.sh

In [208]:
# !chmod 777 $bashFileTmp
# !cd $workDir; \
#     nestrun --template-file $bashFileTmp -d $buildDir --log-file MW0.5-HR-SIP.log -j 14

In [181]:
#%pushnote MW0.5-HR-SIP complete: $buildDir

qSIP


In [ ]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_qSIP.sh'
bashFileTmp

In [ ]:
%%writefile $bashFileTmp
#!/bin/bash


# qSIP
SIPSim qSIP \
    OTU_abs{abs}.txt \
    OTU_abs{abs}_PCR_sub.txt \
    > OTU_abs{abs}_PCR_sub_qSIP.txt
        

# qSIP: atom excess
SIPSim qSIP_atomExcess \
    --np {np} \
    OTU_abs{abs}_PCR_sub_qSIP.txt \
    {exp_design} \
    > OTU_abs{abs}_PCR_sub_qSIP_atom.txt

In [ ]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file qSIP.log -j 2

In [ ]:
%pushnote qSIP complete: $buildDir

deltaBD


In [70]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_dBD.sh'
bashFileTmp


Out[70]:
'/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_dBD.sh'

In [51]:
%%writefile $bashFileTmp
#!/bin/bash

#deltaBD 
SIPSim deltaBD \
    OTU_abs{abs}_PCR_sub.txt \
    {exp_design} \
    > OTU_abs{abs}_PCR_sub_dBD.txt


Writing /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_dBD.sh

In [52]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file deltaBD.log -j 4


2016-04-26 12:46:49,856 * INFO * Template: ./SIPSimRun_dBD.sh
2016-04-26 12:46:49,858 * INFO * [43597] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7
2016-04-26 12:46:49,859 * INFO * [43598] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9
2016-04-26 12:46:49,861 * INFO * [43599] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8
2016-04-26 12:46:49,862 * INFO * [43602] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3
2016-04-26 12:46:53,841 * INFO * [43598] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9 Finished with 0
2016-04-26 12:46:53,843 * INFO * [43713] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10
2016-04-26 12:46:53,892 * INFO * [43597] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7 Finished with 0
2016-04-26 12:46:53,894 * INFO * [43715] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2
2016-04-26 12:46:54,138 * INFO * [43602] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3 Finished with 0
2016-04-26 12:46:54,140 * INFO * [43717] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1
2016-04-26 12:46:54,264 * INFO * [43599] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8 Finished with 0
2016-04-26 12:46:54,266 * INFO * [43741] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4
2016-04-26 12:46:57,432 * INFO * [43715] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2 Finished with 0
2016-04-26 12:46:57,434 * INFO * [43829] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5
2016-04-26 12:46:57,465 * INFO * [43713] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10 Finished with 0
2016-04-26 12:46:57,467 * INFO * [43831] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6
2016-04-26 12:46:57,791 * INFO * [43741] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4 Finished with 0
2016-04-26 12:46:57,793 * INFO * [43845] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7
2016-04-26 12:46:58,229 * INFO * [43717] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1 Finished with 0
2016-04-26 12:46:58,231 * INFO * [43876] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9
2016-04-26 12:47:01,022 * INFO * [43831] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6 Finished with 0
2016-04-26 12:47:01,024 * INFO * [43945] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8
2016-04-26 12:47:01,048 * INFO * [43829] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5 Finished with 0
2016-04-26 12:47:01,050 * INFO * [43947] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3
2016-04-26 12:47:01,757 * INFO * [43876] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9 Finished with 0
2016-04-26 12:47:01,759 * INFO * [43977] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10
2016-04-26 12:47:01,873 * INFO * [43845] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7 Finished with 0
2016-04-26 12:47:01,874 * INFO * [43993] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2
2016-04-26 12:47:04,791 * INFO * [43945] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8 Finished with 0
2016-04-26 12:47:04,793 * INFO * [44061] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1
2016-04-26 12:47:04,799 * INFO * [43947] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3 Finished with 0
2016-04-26 12:47:04,800 * INFO * [44063] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4
2016-04-26 12:47:05,429 * INFO * [43993] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2 Finished with 0
2016-04-26 12:47:05,430 * INFO * [44093] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5
2016-04-26 12:47:05,491 * INFO * [43977] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10 Finished with 0
2016-04-26 12:47:05,493 * INFO * [44095] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6
2016-04-26 12:47:08,385 * INFO * [44063] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4 Finished with 0
2016-04-26 12:47:08,387 * INFO * [44177] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7
2016-04-26 12:47:08,425 * INFO * [44061] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1 Finished with 0
2016-04-26 12:47:08,427 * INFO * [44179] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9
2016-04-26 12:47:08,921 * INFO * [44093] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5 Finished with 0
2016-04-26 12:47:08,923 * INFO * [44207] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8
2016-04-26 12:47:09,346 * INFO * [44095] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6 Finished with 0
2016-04-26 12:47:09,347 * INFO * [44246] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3
2016-04-26 12:47:11,942 * INFO * [44177] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7 Finished with 0
2016-04-26 12:47:11,944 * INFO * [44293] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/10
2016-04-26 12:47:12,033 * INFO * [44179] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9 Finished with 0
2016-04-26 12:47:12,035 * INFO * [44295] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/2
2016-04-26 12:47:12,561 * INFO * [44207] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8 Finished with 0
2016-04-26 12:47:12,563 * INFO * [44324] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/1
2016-04-26 12:47:13,048 * INFO * [44246] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3 Finished with 0
2016-04-26 12:47:13,050 * INFO * [44366] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/4
2016-04-26 12:47:15,479 * INFO * [44293] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/10 Finished with 0
2016-04-26 12:47:15,483 * INFO * [44437] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/5
2016-04-26 12:47:15,666 * INFO * [44295] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/2 Finished with 0
2016-04-26 12:47:15,668 * INFO * [44439] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/6
2016-04-26 12:47:16,153 * INFO * [44324] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/1 Finished with 0
2016-04-26 12:47:16,155 * INFO * [44468] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/7
2016-04-26 12:47:16,749 * INFO * [44366] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/4 Finished with 0
2016-04-26 12:47:16,750 * INFO * [44511] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/9
2016-04-26 12:47:19,120 * INFO * [44437] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/5 Finished with 0
2016-04-26 12:47:19,121 * INFO * [44553] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/8
2016-04-26 12:47:19,784 * INFO * [44468] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/7 Finished with 0
2016-04-26 12:47:19,785 * INFO * [44569] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/3
2016-04-26 12:47:19,844 * INFO * [44439] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/6 Finished with 0
2016-04-26 12:47:19,846 * INFO * [44571] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/10
2016-04-26 12:47:20,358 * INFO * [44511] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/9 Finished with 0
2016-04-26 12:47:20,360 * INFO * [44612] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/2
2016-04-26 12:47:22,883 * INFO * [44553] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/8 Finished with 0
2016-04-26 12:47:22,884 * INFO * [44669] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/1
2016-04-26 12:47:23,378 * INFO * [44569] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/3 Finished with 0
2016-04-26 12:47:23,379 * INFO * [44684] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/4
2016-04-26 12:47:23,469 * INFO * [44571] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/10 Finished with 0
2016-04-26 12:47:23,471 * INFO * [44687] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/5
2016-04-26 12:47:23,935 * INFO * [44612] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/2 Finished with 0
2016-04-26 12:47:23,937 * INFO * [44729] Started ./SIPSimRun_dBD.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/10/6
**OUTPUT MUTED**

In [53]:
%pushnote deltaBD complete: $buildDir

Making confusion matrices


In [182]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_cMtx.sh'
bashFileTmp


Out[182]:
'/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_cMtx.sh'

In [183]:
%%writefile $bashFileTmp
#!/bin/bash

# HR-SIP
SIPSimR DESeq2_confuseMtx \
    --libs 2,4,6 \
    --padj {padj} \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_filt_DESeq2

# HR-SIP multiple 'heavy' BD windows
SIPSimR DESeq2_confuseMtx \
    --libs 2,4,6 \
    --padj {padj} \
    -o DESeq2_multi-cMtx \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_filt_MW_DESeq2  
    
# HR-SIP multiple 'heavy' BD windows: l2fc_cut=0.5
SIPSimR DESeq2_confuseMtx \
    --libs 2,4,6 \
    --padj {padj} \
    -o DESeq2_l2fcCut0.5-multi-cMtx \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_filt_MW0.5_DESeq2  
        
    
# qSIP    
SIPSimR qSIP_confuseMtx \
    --libs 2,4,6 \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_qSIP_atom.txt

# heavy-SIP    
SIPSimR heavy_confuseMtx \
    --libs 2,4,6 \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub.txt


Overwriting /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_cMtx.sh

In [184]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file cMtx.log -j 14


2016-05-16 07:50:24,913 * INFO * Template: ./SIPSimRun_cMtx.sh
2016-05-16 07:50:24,916 * INFO * [16316] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7
2016-05-16 07:50:24,919 * INFO * [16317] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9
2016-05-16 07:50:24,922 * INFO * [16319] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8
2016-05-16 07:50:24,924 * INFO * [16321] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3
2016-05-16 07:50:24,927 * INFO * [16323] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10
2016-05-16 07:50:24,928 * INFO * [16325] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2
2016-05-16 07:50:24,930 * INFO * [16327] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1
2016-05-16 07:50:24,932 * INFO * [16330] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4
2016-05-16 07:50:24,934 * INFO * [16331] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5
2016-05-16 07:50:24,935 * INFO * [16333] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6
2016-05-16 07:50:24,936 * INFO * [16335] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/7
2016-05-16 07:50:24,938 * INFO * [16337] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/9
2016-05-16 07:50:24,940 * INFO * [16340] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/8
2016-05-16 07:50:24,941 * INFO * [16341] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/3
2016-05-16 07:50:44,615 * INFO * [16317] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9 Finished with 0
2016-05-16 07:50:44,616 * INFO * [17844] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/10
2016-05-16 07:50:44,752 * INFO * [16319] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8 Finished with 0
2016-05-16 07:50:44,754 * INFO * [17852] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/2
2016-05-16 07:50:44,780 * INFO * [16335] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/7 Finished with 0
2016-05-16 07:50:44,782 * INFO * [17854] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/1
2016-05-16 07:50:44,810 * INFO * [16323] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10 Finished with 0
2016-05-16 07:50:44,812 * INFO * [17858] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/4
2016-05-16 07:50:44,877 * INFO * [16340] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/8 Finished with 0
2016-05-16 07:50:44,878 * INFO * [17862] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/5
2016-05-16 07:50:44,890 * INFO * [16337] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/9 Finished with 0
2016-05-16 07:50:44,892 * INFO * [17864] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/6
2016-05-16 07:50:44,962 * INFO * [16327] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1 Finished with 0
2016-05-16 07:50:44,964 * INFO * [17879] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/7
2016-05-16 07:50:45,177 * INFO * [16321] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3 Finished with 0
2016-05-16 07:50:45,179 * INFO * [17920] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/9
2016-05-16 07:50:45,416 * INFO * [16316] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7 Finished with 0
2016-05-16 07:50:45,418 * INFO * [17993] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/8
2016-05-16 07:50:45,608 * INFO * [16330] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4 Finished with 0
2016-05-16 07:50:45,610 * INFO * [18030] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/3
2016-05-16 07:50:45,673 * INFO * [16341] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/3 Finished with 0
2016-05-16 07:50:45,675 * INFO * [18032] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/10
2016-05-16 07:50:45,976 * INFO * [16331] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5 Finished with 0
2016-05-16 07:50:45,977 * INFO * [18070] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/2
2016-05-16 07:50:46,014 * INFO * [16325] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2 Finished with 0
2016-05-16 07:50:46,016 * INFO * [18087] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/1
2016-05-16 07:50:46,367 * INFO * [16333] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6 Finished with 0
2016-05-16 07:50:46,369 * INFO * [18129] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/4
2016-05-16 07:51:03,471 * INFO * [17858] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/4 Finished with 0
2016-05-16 07:51:03,472 * INFO * [19425] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/5
2016-05-16 07:51:03,481 * INFO * [17879] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/7 Finished with 0
2016-05-16 07:51:03,482 * INFO * [19427] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/6
2016-05-16 07:51:03,579 * INFO * [17844] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/10 Finished with 0
2016-05-16 07:51:03,580 * INFO * [19433] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7
2016-05-16 07:51:03,614 * INFO * [17864] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/6 Finished with 0
2016-05-16 07:51:03,616 * INFO * [19435] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9
2016-05-16 07:51:03,703 * INFO * [17854] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/1 Finished with 0
2016-05-16 07:51:03,705 * INFO * [19439] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8
2016-05-16 07:51:03,792 * INFO * [17852] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/2 Finished with 0
2016-05-16 07:51:03,794 * INFO * [19449] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3
2016-05-16 07:51:03,835 * INFO * [17862] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/5 Finished with 0
2016-05-16 07:51:03,837 * INFO * [19469] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10
2016-05-16 07:51:03,943 * INFO * [17920] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/9 Finished with 0
2016-05-16 07:51:03,945 * INFO * [19499] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2
2016-05-16 07:51:04,089 * INFO * [17993] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/8 Finished with 0
2016-05-16 07:51:04,091 * INFO * [19530] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1
2016-05-16 07:51:04,699 * INFO * [18032] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/10 Finished with 0
2016-05-16 07:51:04,701 * INFO * [19632] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4
2016-05-16 07:51:04,739 * INFO * [18030] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/3 Finished with 0
2016-05-16 07:51:04,741 * INFO * [19636] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5
2016-05-16 07:51:04,799 * INFO * [18087] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/1 Finished with 0
2016-05-16 07:51:04,801 * INFO * [19638] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6
2016-05-16 07:51:06,448 * INFO * [18070] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/2 Finished with 0
2016-05-16 07:51:06,450 * INFO * [19699] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7
2016-05-16 07:51:06,996 * INFO * [18129] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/4 Finished with 0
2016-05-16 07:51:06,997 * INFO * [19721] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9
2016-05-16 07:51:22,678 * INFO * [19427] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/6 Finished with 0
2016-05-16 07:51:22,680 * INFO * [21034] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8
2016-05-16 07:51:22,690 * INFO * [19449] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3 Finished with 0
2016-05-16 07:51:22,692 * INFO * [21052] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3
2016-05-16 07:51:22,842 * INFO * [19425] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/5 Finished with 0
2016-05-16 07:51:22,844 * INFO * [21249] Started ./SIPSimRun_cMtx.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/10
**OUTPUT MUTED**

Aggregating the confusion matrix data


In [13]:
def agg_cMtx(prefix):
    # all data
    x = prefix + '-cMtx_data.txt'
    !nestagg delim \
       -d $buildDir \
       -k shared_perc,perm_perc,rep \
       -o $x \
       --tab \
       $x

    # overall
    x = prefix + '-cMtx_overall.txt'
    !nestagg delim \
        -d $buildDir \
        -k shared_perc,perm_perc,rep \
        -o $x \
        --tab \
        $x

    # by class
    x = prefix + '-cMtx_byClass.txt'
    !nestagg delim \
        -d $buildDir \
        -k shared_perc,perm_perc,rep \
        -o $x \
        --tab \
        $x
        
agg_cMtx('DESeq2')
agg_cMtx('DESeq2_multi')
agg_cMtx('DESeq2_l2fcCut0.5-multi')
agg_cMtx('qSIP') 
agg_cMtx('heavy')

In [186]:
%pushnote microBetaDiv complete!

Calculating pre-fractionation comm beta-div indices


In [119]:
bashFileTmp = os.path.splitext(bashFile)[0] + '_betaDiv.sh'
bashFileTmp


Out[119]:
'/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_betaDiv.sh'

In [120]:
%%writefile $bashFileTmp
#!/bin/bash

# beta diversity
SIPSimR comm_beta_div \
    comm.txt \
    > comm_betaDiv.txt


Writing /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/SIPSimRun_betaDiv.sh

In [121]:
!chmod 777 $bashFileTmp
!cd $workDir; \
    nestrun --template-file $bashFileTmp -d $buildDir --log-file cMtx.log -j 14


2016-06-16 14:14:01,784 * INFO * Template: ./SIPSimRun_betaDiv.sh
2016-06-16 14:14:01,805 * INFO * [86837] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7
2016-06-16 14:14:01,814 * INFO * [86839] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9
2016-06-16 14:14:01,816 * INFO * [86840] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8
2016-06-16 14:14:01,828 * INFO * [86843] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3
2016-06-16 14:14:01,830 * INFO * [86845] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10
2016-06-16 14:14:01,837 * INFO * [86847] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2
2016-06-16 14:14:01,840 * INFO * [86849] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1
2016-06-16 14:14:01,841 * INFO * [86850] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4
2016-06-16 14:14:01,850 * INFO * [86853] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5
2016-06-16 14:14:01,852 * INFO * [86854] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6
2016-06-16 14:14:01,854 * INFO * [86857] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/7
2016-06-16 14:14:01,872 * INFO * [86859] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/9
2016-06-16 14:14:01,874 * INFO * [86860] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/8
2016-06-16 14:14:01,880 * INFO * [86863] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/3
2016-06-16 14:14:07,620 * INFO * [86843] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/3 Finished with 0
2016-06-16 14:14:07,640 * INFO * [87157] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/10
2016-06-16 14:14:07,641 * INFO * [86837] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/7 Finished with 0
2016-06-16 14:14:07,673 * INFO * [87161] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/2
2016-06-16 14:14:07,673 * INFO * [86840] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/8 Finished with 0
2016-06-16 14:14:07,686 * INFO * [87163] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/1
2016-06-16 14:14:07,686 * INFO * [86839] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/9 Finished with 0
2016-06-16 14:14:07,700 * INFO * [87165] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/4
2016-06-16 14:14:07,700 * INFO * [86845] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/10 Finished with 0
2016-06-16 14:14:07,715 * INFO * [87167] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/5
2016-06-16 14:14:07,715 * INFO * [86847] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/2 Finished with 0
2016-06-16 14:14:07,726 * INFO * [87169] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/6
2016-06-16 14:14:07,726 * INFO * [86849] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/1 Finished with 0
2016-06-16 14:14:07,738 * INFO * [87171] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/7
2016-06-16 14:14:07,738 * INFO * [86850] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/4 Finished with 0
2016-06-16 14:14:07,751 * INFO * [87173] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/9
2016-06-16 14:14:07,751 * INFO * [86853] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/5 Finished with 0
2016-06-16 14:14:07,766 * INFO * [87175] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/8
2016-06-16 14:14:07,766 * INFO * [86854] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/20/6 Finished with 0
2016-06-16 14:14:07,769 * INFO * [87177] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/3
2016-06-16 14:14:07,770 * INFO * [86857] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/7 Finished with 0
2016-06-16 14:14:07,773 * INFO * [87179] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/10
2016-06-16 14:14:07,774 * INFO * [86859] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/9 Finished with 0
2016-06-16 14:14:07,775 * INFO * [87181] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/2
2016-06-16 14:14:07,776 * INFO * [86860] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/8 Finished with 0
2016-06-16 14:14:07,785 * INFO * [87183] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/1
2016-06-16 14:14:07,786 * INFO * [86863] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/3 Finished with 0
2016-06-16 14:14:07,795 * INFO * [87185] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/4
2016-06-16 14:14:10,437 * INFO * [87157] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/10 Finished with 0
2016-06-16 14:14:10,454 * INFO * [87455] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/5
2016-06-16 14:14:10,555 * INFO * [87167] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/5 Finished with 0
2016-06-16 14:14:10,586 * INFO * [87459] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/6
2016-06-16 14:14:10,885 * INFO * [87175] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/8 Finished with 0
2016-06-16 14:14:10,898 * INFO * [87469] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/7
2016-06-16 14:14:10,933 * INFO * [87185] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/4 Finished with 0
2016-06-16 14:14:10,946 * INFO * [87472] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9
2016-06-16 14:14:10,946 * INFO * [87161] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/2 Finished with 0
2016-06-16 14:14:10,948 * INFO * [87475] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8
2016-06-16 14:14:10,971 * INFO * [87179] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/10 Finished with 0
2016-06-16 14:14:10,981 * INFO * [87488] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/3
2016-06-16 14:14:11,006 * INFO * [87165] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/4 Finished with 0
2016-06-16 14:14:11,009 * INFO * [87501] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10
2016-06-16 14:14:11,216 * INFO * [87163] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/1 Finished with 0
2016-06-16 14:14:11,220 * INFO * [87512] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/2
2016-06-16 14:14:11,306 * INFO * [87183] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/1 Finished with 0
2016-06-16 14:14:11,308 * INFO * [87540] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/1
2016-06-16 14:14:11,385 * INFO * [87171] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/7 Finished with 0
2016-06-16 14:14:11,388 * INFO * [87559] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/4
2016-06-16 14:14:11,426 * INFO * [87177] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/3 Finished with 0
2016-06-16 14:14:11,429 * INFO * [87585] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/5
2016-06-16 14:14:11,429 * INFO * [87169] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/30/6 Finished with 0
2016-06-16 14:14:11,431 * INFO * [87588] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/6
2016-06-16 14:14:11,454 * INFO * [87173] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/9 Finished with 0
2016-06-16 14:14:11,466 * INFO * [87600] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/7
2016-06-16 14:14:11,669 * INFO * [87181] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/50/2 Finished with 0
2016-06-16 14:14:11,671 * INFO * [87652] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/9
2016-06-16 14:14:13,789 * INFO * [87475] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/8 Finished with 0
2016-06-16 14:14:13,791 * INFO * [87781] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/8
2016-06-16 14:14:13,819 * INFO * [87472] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/9 Finished with 0
2016-06-16 14:14:13,839 * INFO * [87783] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/3
2016-06-16 14:14:13,859 * INFO * [87501] /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/15/10 Finished with 0
2016-06-16 14:14:13,861 * INFO * [87785] Started ./SIPSimRun_betaDiv.sh in /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/80/0/10
**OUTPUT MUTED**

--End of simulation--


Plotting results


In [220]:
F = os.path.join(buildDir, '*-cMtx_byClass.txt')
files = glob.glob(F)
files


Out[220]:
['/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/DESeq2-cMtx_byClass.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/DESeq2_l2fcCut0.5-multi-cMtx_byClass.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/qSIP-cMtx_byClass.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/heavy-cMtx_byClass.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/DESeq2_multi-cMtx_byClass.txt']

In [221]:
%%R -i files

df_byClass = list()
for (f in files){
    ff = strsplit(f, '/') %>% unlist
    fff = ff[length(ff)]
    df_byClass[[fff]] = read.delim(f, sep='\t')
}

df_byClass = do.call(rbind, df_byClass)
df_byClass$file = gsub('\\.[0-9]+$', '', rownames(df_byClass))
df_byClass$method = gsub('-.+', '', df_byClass$file)
rownames(df_byClass) = 1:nrow(df_byClass)
df_byClass = filter(df_byClass, perm_perc <= 20)

df_byClass %>% head(n=3)


  library      variables    values shared_perc perm_perc rep
1       2    Sensitivity 0.5940594          80        20   7
2       2    Specificity 0.9168421          80        20   7
3       2 Pos Pred Value 0.4316547          80        20   7
                     file method
1 DESeq2-cMtx_byClass.txt DESeq2
2 DESeq2-cMtx_byClass.txt DESeq2
3 DESeq2-cMtx_byClass.txt DESeq2

In [222]:
%%R
# renaming method
rename = data.frame(method = c('DESeq2', 'DESeq2_multi', 'heavy', 'qSIP'), 
                   method_new = c('HR-SIP', 'MW-HR-SIP', 'Heavy-SIP', 'q-SIP'))

df_byClass = inner_join(df_byClass, rename, c('method'='method')) %>%
    select(-method) %>%
    rename('method' = method_new) 

df_byClass %>% head(n=3)


  library      variables    values shared_perc perm_perc rep
1       2    Sensitivity 0.5940594          80        20   7
2       2    Specificity 0.9168421          80        20   7
3       2 Pos Pred Value 0.4316547          80        20   7
                     file method
1 DESeq2-cMtx_byClass.txt HR-SIP
2 DESeq2-cMtx_byClass.txt HR-SIP
3 DESeq2-cMtx_byClass.txt HR-SIP

In [223]:
%%R -w 800 -h 550
# summarize by SIPSim rep & library rep
df_byClass.s = df_byClass %>%
    group_by(method, shared_perc, perm_perc, variables) %>%
    summarize(mean_value = mean(values),
              sd_value = sd(values))

# plotting
ggplot(df_byClass.s, aes(variables, mean_value, color=method,
                         ymin=mean_value-sd_value,
                         ymax=mean_value+sd_value)) +
    geom_pointrange(alpha=0.8, size=0.2) +
    labs(y='Value') +
    facet_grid(shared_perc ~ perm_perc) +
    theme_bw() +
    theme(
        text = element_text(size=16),
        axis.title.x = element_blank(),
        axis.text.x = element_text(angle=45, hjust=1)
    )



In [224]:
%%R -w 800 -h 600
# summarize by SIPSim rep & library rep
vars = c('Balanced Accuracy', 'Sensitivity', 'Specificity')
df_byClass.s.f = df_byClass.s %>%
    filter(variables %in% vars) 

# plotting
ggplot(df_byClass.s.f, aes(variables, mean_value, fill=method,
                         ymin=mean_value-sd_value,
                         ymax=mean_value+sd_value)) +
    #geom_pointrange(alpha=0.8, size=0.2) +
    geom_bar(stat='identity', position='dodge', width=0.8) +
    geom_errorbar(stat='identity', position='dodge', width=0.8) +
    scale_y_continuous(breaks=seq(0, 1, 0.2)) +
    labs(y='Value') +
    facet_grid(shared_perc ~ perm_perc) +
    theme_bw() +
    theme(
        text = element_text(size=16),
        axis.title.x = element_blank(),
        axis.text.x = element_text(angle=50, hjust=1)
    )



In [225]:
%%R -w 800 -h 450
# summarize by SIPSim rep & library rep
vars = c('Balanced Accuracy', 'Sensitivity', 'Specificity')
df_byClass.s.f = df_byClass.s %>%
    filter(variables %in% vars) %>%
    ungroup() %>%
    mutate(perm_perc = perm_perc %>% as.character,
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric))

# plotting
p.pnt = ggplot(df_byClass.s.f, aes(shared_perc, mean_value, 
                           color=perm_perc, 
                           ymin=mean_value-sd_value,
                           ymax=mean_value+sd_value)) +
    geom_point(alpha=0.8) +
    geom_linerange(alpha=0.8, size=0.5) +
    geom_line() +
    scale_color_discrete('% of rank\nabundances\npermuted') +
    labs(x='% taxa shared among pre-fractionation communities') +
    facet_grid(variables ~ method, scale='free_y') +
    theme_bw() +
    theme(
        text = element_text(size=16),
        axis.title.y = element_blank()
    )
p.pnt


Notes

  • The lowered specificity for (MW-)HR-SIP is mainly due to community structure differences between control and treatment gradients.

In [226]:
%%R 
outFile = 'microBetaDiv_acc.pdf'
ggsave(outFile, p.pnt, width=10, height=6)
cat('File written:', file.path(getwd(), outFile), '\n')


File written: /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/microBetaDiv_acc.pdf 

Plotting fraction of false positives (of total estimated positives)

  • FP / (FP + TP)

In [51]:
F = os.path.join(buildDir, '*-cMtx_data.txt')
files = glob.glob(F)
files


Out[51]:
['/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/qSIP-cMtx_data.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/DESeq2-cMtx_data.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/DESeq2_multi-cMtx_data.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/heavy-cMtx_data.txt',
 '/home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/DESeq2_l2fcCut0.5-multi-cMtx_data.txt']

In [52]:
%%R -i files
# calling TP,FP,TN,FN
Clsfy = function(known, pred){
    #print(known == TRUE)
    #print(pred == TRUE)
    if(known == TRUE & pred == TRUE){
        return('TP')
    } else
    if(known == TRUE & pred == FALSE){
        return('FN')
    } else
    if(known == FALSE & pred == FALSE){
        return('TN')
    } else
    if(known == FALSE & pred == TRUE){
        return('FP')
    } else {
        stop('logic error')
    }        
}

# loading data
df_data = list()
for (f in files){
    ff = strsplit(f, '/') %>% unlist
    fff = ff[length(ff)]
    df_data[[fff]] = read.delim(f, sep='\t') %>%
        rowwise() %>%
        mutate(clsfy = Clsfy(incorp.known, incorp.pred)) %>%
        dplyr::select(library, taxon, incorp.known, incorp.pred,
                       clsfy, shared_perc, perm_perc, rep) %>%
        group_by(library, shared_perc, perm_perc, rep, clsfy) %>%
        summarize(n_clsfy = n()) %>%
        ungroup()
}

# combining data
df_data = do.call(rbind, df_data)
df_data$file = gsub('\\.[0-9]+$', '', rownames(df_data))
df_data$method = gsub('-.+', '', df_data$file)
rownames(df_data) = 1:nrow(df_data)

# status
df_data %>% head(n=3)


Source: local data frame [3 x 8]

  library shared_perc perm_perc   rep clsfy n_clsfy               file method
    (int)       (int)     (int) (int) (chr)   (int)              (chr)  (chr)
1       2          80         0     1    FN      40 qSIP-cMtx_data.txt   qSIP
2       2          80         0     1    FP      63 qSIP-cMtx_data.txt   qSIP
3       2          80         0     1    TN     930 qSIP-cMtx_data.txt   qSIP

In [53]:
%%R
# renaming method
rename = data.frame(method = c('DESeq2', 'DESeq2_multi', 'heavy', 'qSIP'), 
                   method_new = c('HR-SIP', 'MW-HR-SIP', 'Heavy-SIP', 'q-SIP'))

df_data = inner_join(df_data, rename, c('method'='method')) %>%
    select(-method) %>%
    rename('method' = method_new) 

# status
df_data %>% head(n=3)


Source: local data frame [3 x 8]

  library shared_perc perm_perc   rep clsfy n_clsfy               file method
    (int)       (int)     (int) (int) (chr)   (int)              (chr) (fctr)
1       2          80         0     1    FN      40 qSIP-cMtx_data.txt  q-SIP
2       2          80         0     1    FP      63 qSIP-cMtx_data.txt  q-SIP
3       2          80         0     1    TN     930 qSIP-cMtx_data.txt  q-SIP

In [54]:
%%R -w 800 -h 250
as.Num = function(x) x %>% as.character %>% as.numeric

# summarizing
df_data_s = df_data %>%
    filter(perm_perc <= 20) %>%
    spread(clsfy, n_clsfy, fill=0) %>%
    mutate(FP = FP %>% as.Num,
           TP = TP %>% as.Num,
           FN = FN %>% as.Num,
           FN = ifelse(is.na(FN), 0, FN),
           FP = ifelse(is.na(FP), 0, FP),
           TP = ifelse(is.na(TP), 0, TP))%>%
    rowwise %>% 
    mutate(FP = ifelse(FP / (FP+TN) < 0.01, 0, FP), 
           FP_frac = FP / (FP+TP),
           FP_frac = ifelse(is.na(FP_frac), 0, FP_frac)) %>%
    group_by(method, shared_perc, perm_perc) %>%
    summarize(mean_FP_frac = mean(FP_frac, na.rm=TRUE),
              sd_FP_frac = sd(FP_frac, na.rm=TRUE)) %>%
    ungroup() %>%
    mutate(perm_perc = perm_perc %>% as.character,
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric))
#df_data_s %>% tail %>% as.data.frame

y.lab = expression(frac('False incorporators', 'Total incorporators'))    
   
    
# plotting
p.FP = ggplot(df_data_s, aes(shared_perc, mean_FP_frac, 
                           color=perm_perc, group=perm_perc,
                           ymin=mean_FP_frac-sd_FP_frac,
                           ymax=mean_FP_frac+sd_FP_frac)) +
    geom_point(alpha=0.8) +
    geom_linerange(alpha=0.8, size=0.5) +
    geom_line() +
    scale_y_continuous(limits=c(-0.16,1.00)) +
    scale_color_discrete('% of rank\nabundances\npermuted') +
    labs(x='% taxa shared among pre-fractionation communities', y=y.lab) +    
    facet_grid( ~ method, scales='free_y') +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )
p.FP


Plot number of false positives

  • emphasize the issue of specificity

In [115]:
%%R -w 800 -h 250
# theoretical number of taxa
df.ntaxa = data.frame(
    'variables' = rep('Specificity', 3),
    'ntaxa' = c(500, 900, 2000)
)

df_byClass.j = inner_join(df_byClass, df.ntaxa, c('variables'='variables')) %>%
    mutate(nFP = (1-values) * ntaxa) %>%
    group_by(method, shared_perc, perm_perc, ntaxa) %>%
    summarize(mean_nFP = mean(nFP, na.rm=TRUE),
              sd_nFP = sd(nFP, na.rm=TRUE)) %>%
    ungroup() %>%
    mutate(perm_perc = perm_perc %>% as.character,
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric),
           ntaxa = ntaxa %>% as.character) %>%
    filter(ntaxa == 900)

# plotting
p.FP = ggplot(df_byClass.j, aes(shared_perc, mean_nFP, 
                           color=perm_perc, group=perm_perc,
                           ymin=mean_nFP-sd_nFP,
                           ymax=mean_nFP+sd_nFP)) +
    geom_point(alpha=0.8) +
    geom_linerange(alpha=0.8, size=0.5) +
    geom_line() +
    scale_color_discrete('% of rank\nabundances\npermuted') +
    labs(x='% taxa shared replicate pre-fractionation communities',
         y='False incorporators') +
    facet_grid( ~ method, scales='free_y') +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )
p.FP



In [265]:
%%R
# max number of FP
df_byClass.j %>% 
    filter(method=='q-SIP') %>%
    arrange(-mean_nFP) %>%
    head(n=3) %>% print

# min number of FP
df_byClass.j %>% 
    filter(method=='q-SIP') %>%
    arrange(-mean_nFP) %>%
    tail(n=3)


Source: local data frame [3 x 6]

  method shared_perc perm_perc ntaxa mean_nFP   sd_nFP
  (fctr)       (int)    (fctr) (chr)    (dbl)    (dbl)
1  q-SIP         100        15   900 127.9573 49.06887
2  q-SIP         100        10   900 125.0310 63.24409
3  q-SIP         100        20   900 117.2994 36.14826
Source: local data frame [3 x 6]

  method shared_perc perm_perc ntaxa mean_nFP   sd_nFP
  (fctr)       (int)    (fctr) (chr)    (dbl)    (dbl)
1  q-SIP          85         5   900 67.39419 28.61702
2  q-SIP          80         0   900 65.14412 31.93217
3  q-SIP          85         0   900 63.58130 36.44136

Combined figure


In [116]:
%%R
g_legend = function(a.gplot){ 
  tmp = ggplot_gtable(ggplot_build(a.gplot)) 
  leg = which(sapply(tmp$grobs, function(x) x$name) == "guide-box") 
  legend = tmp$grobs[[leg]] 
  return(legend)
  }
                     
p.leg = g_legend(p.pnt)

In [117]:
%%R -w 800 -h 650

p.pnt.e = p.pnt + theme(legend.position='none')
p.FP.e = p.FP + theme(legend.position='none')

# p.comb = cowplot::ggdraw() +
#     cowplot::draw_plot(p.pnt.e, 0.03, 0.3, 0.875, 0.7) +
#     cowplot::draw_plot(p.FP.e, 0, 0, 0.88, 0.3) +
#     cowplot::draw_plot(p.leg, 0.92, 0.38, 0.08, 0.3) +
#     cowplot::draw_plot_label(c('A)', 'B)'), c(0, 0), c(1, 0.31))
# p.comb

p.comb = cowplot::ggdraw() +
    geom_rect(aes(xmin=0, ymin=0, xmax=1, ymax=1), color='white', fill='white') +
#    cowplot::draw_plot(p.pnt.e, 0.07, 0.31, 0.8, 0.69) +
    cowplot::draw_plot(p.pnt.e, 0.03, 0.31, 0.84, 0.69) +
    cowplot::draw_plot(p.FP.e, 0, 0, 0.845, 0.3) +
    cowplot::draw_plot(p.leg, 0.89, 0.38, 0.08, 0.3) +
    cowplot::draw_plot_label(c('A)', 'B)'), c(0, 0), c(1, 0.32))
p.comb



In [118]:
%%R
outFile = 'microBetaDiv_acc-FP.pdf'
ggsave(outFile, p.comb, width=10, height=8.65)
cat('File written:', file.path(getwd(), outFile), '\n')


File written: /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/microBetaDiv_acc-FP.pdf 

atom excess estimates

qSIP


In [261]:
atomX_files = glob.glob('*/*/*/*_atom.txt')
len(atomX_files)


Out[261]:
400

In [262]:
%%R -i atomX_files

df_atomX = list()
for(F in atomX_files){
    tmp = read.delim(F, sep='\t') 
    FF = strsplit(F, '/') %>% unlist
    tmp$shared_perc = FF[1]
    tmp$perm_perc = FF[2]
    tmp$rep = FF[3]
    tmp$file = FF[4]
    df_atomX[[F]] = tmp
}

df_atomX = do.call(rbind, df_atomX)
rownames(df_atomX) = 1:nrow(df_atomX)
df_atomX %>% head(n=3)


                                 taxon  control treatment       BD_diff
1       Acaryochloris_marina_MBIC11017 1.708033  1.709418  0.0013851679
2 Acetobacter_pasteurianus_IFO_3283-03 1.712330  1.712616  0.0002866604
3       Acetobacterium_woodii_DSM_1030 1.704179  1.703843 -0.0003364652
  control_GC control_MW treatment_max_MW treatment_MW atom_fraction_excess
1  0.7421776   308.0591         317.6635     308.3089          0.025722669
2  0.7936284   308.0846         317.6634     308.1362          0.005324613
3  0.6960235   308.0362         317.6637     307.9754         -0.006246878
   atom_CI_low atom_CI_high shared_perc perm_perc rep
1 -0.007952873   0.05800202          80        20   7
2 -0.013487728   0.02572328          80        20   7
3 -0.025040726   0.01425911          80        20   7
                              file
1 OTU_abs1e9_PCR_sub_qSIP_atom.txt
2 OTU_abs1e9_PCR_sub_qSIP_atom.txt
3 OTU_abs1e9_PCR_sub_qSIP_atom.txt

In [264]:
BDshift_files = glob.glob('*/*/*/BD-shift_stats.txt')
len(BDshift_files)


Out[264]:
400

In [265]:
%%R -i BDshift_files

df_shift = list()
for(F in BDshift_files){
    tmp = read.delim(F, sep='\t') 
    FF = strsplit(F, '/') %>% unlist
    tmp$shared_perc = FF[1]
    tmp$perm_perc = FF[2]
    tmp$rep = FF[3]
    tmp$file = FF[4]
    df_shift[[F]] = tmp
}

df_shift = do.call(rbind, df_shift)
rownames(df_shift) = 1:nrow(df_shift)

df_shift = df_shift %>%
    filter(library %in% c(2,4,6)) %>%
    group_by(taxon, shared_perc, perm_perc, rep) %>%
    summarize(median = median(median)) %>%
    ungroup() %>%
    rename('median_true_BD_shift' = median)

df_shift %>% head(n=3)


Source: local data frame [3 x 5]

                           taxon shared_perc perm_perc   rep
                          (fctr)       (chr)     (chr) (chr)
1 Acaryochloris_marina_MBIC11017         100         0     1
2 Acaryochloris_marina_MBIC11017         100         0    10
3 Acaryochloris_marina_MBIC11017         100         0     2
Variables not shown: median_true_BD_shift (dbl)

In [266]:
%%R
as.Num = function(x) x %>% as.character %>% as.numeric

df.j = inner_join(df_atomX, df_shift, c('taxon' = 'taxon',
                                       'shared_perc'='shared_perc',
                                       'perm_perc'='perm_perc',
                                       'rep'='rep')) %>%
    mutate(shared_perc = shared_perc %>% as.Num,
           perm_perc = perm_perc %>% as.Num) %>%
    filter(!is.na(BD_diff),
           perm_perc <= 20) %>%
    mutate(true_incorporator = ifelse(median_true_BD_shift > 0.002, TRUE, FALSE),
           true_atom_fraction_excess = median_true_BD_shift / 0.036,
           atom_fraction_excess = ifelse(is.na(atom_CI_low), 0, atom_fraction_excess))
df.j %>% head(n=3)


                                 taxon  control treatment       BD_diff
1       Acaryochloris_marina_MBIC11017 1.708033  1.709418  0.0013851679
2 Acetobacter_pasteurianus_IFO_3283-03 1.712330  1.712616  0.0002866604
3       Acetobacterium_woodii_DSM_1030 1.704179  1.703843 -0.0003364652
  control_GC control_MW treatment_max_MW treatment_MW atom_fraction_excess
1  0.7421776   308.0591         317.6635     308.3089          0.025722669
2  0.7936284   308.0846         317.6634     308.1362          0.005324613
3  0.6960235   308.0362         317.6637     307.9754         -0.006246878
   atom_CI_low atom_CI_high shared_perc perm_perc rep
1 -0.007952873   0.05800202          80        20   7
2 -0.013487728   0.02572328          80        20   7
3 -0.025040726   0.01425911          80        20   7
                              file median_true_BD_shift true_incorporator
1 OTU_abs1e9_PCR_sub_qSIP_atom.txt                    0             FALSE
2 OTU_abs1e9_PCR_sub_qSIP_atom.txt                    0             FALSE
3 OTU_abs1e9_PCR_sub_qSIP_atom.txt                    0             FALSE
  true_atom_fraction_excess
1                         0
2                         0
3                         0

In [267]:
%%R
# free memory
df_shift = df_atomX = NULL

# saving dataframe (if needed)
df.j.qSIP = df.j

In [268]:
%%R -h 250 
# difference between true and estimated
df.j.dis.qSIP = df.j %>%
    mutate(delta_excess = atom_fraction_excess * 100 - true_atom_fraction_excess * 100) %>%
    group_by(shared_perc, perm_perc, true_incorporator) %>%
    summarize(mean_delta_excess = mean(delta_excess),
              sd_delta_excess = sd(delta_excess)) %>%
    ungroup() %>%
    mutate(shared_perc = shared_perc %>% reorder(shared_perc %>% as.numeric),
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric)) %>%
    filter(true_incorporator == TRUE)
   
# plotting
ggplot(df.j.dis.qSIP, aes(shared_perc, mean_delta_excess, 
                      color=perm_perc, group=perm_perc,
                      ymin=mean_delta_excess-sd_delta_excess,
                     ymax=mean_delta_excess+sd_delta_excess)) +
    geom_point() +
    geom_linerange(alpha=0.5, size=0.8) +
    geom_line() +
    scale_color_discrete('% taxa ranks\npermuted') +
    labs(x='% taxa shared', y='atom % excess\n(truth - estimate)') +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )


deltaBD


In [269]:
dBD_files = glob.glob('*/*/*/*_dBD.txt')
len(dBD_files)


Out[269]:
400

In [270]:
%%R -i dBD_files

df_dBD = list()
for(F in dBD_files){
    tmp = read.delim(F, sep='\t') 
    FF = strsplit(F, '/') %>% unlist
    tmp$shared_perc = FF[1]
    tmp$perm_perc = FF[2]
    tmp$rep = FF[3]
    tmp$file = FF[4]
    df_dBD[[F]] = tmp
}

df_dBD = do.call(rbind, df_dBD)
rownames(df_dBD) = 1:nrow(df_dBD)
df_dBD %>% head(n=3)


                                 taxon mean_CM_control mean_CM_treatment
1       Acaryochloris_marina_MBIC11017        1.714107          1.711671
2 Acetobacter_pasteurianus_IFO_3283-03        1.717515          1.714771
3       Acetobacterium_woodii_DSM_1030        1.712993          1.702921
  stdev_CM_control stdev_CM_treatment     delta_BD shared_perc perm_perc rep
1      0.002785874       0.0042661210 -0.002435657          80        20   7
2      0.000842784       0.0019431914 -0.002743521          80        20   7
3      0.001254314       0.0009522154 -0.010071941          80        20   7
                        file
1 OTU_abs1e9_PCR_sub_dBD.txt
2 OTU_abs1e9_PCR_sub_dBD.txt
3 OTU_abs1e9_PCR_sub_dBD.txt

In [271]:
BDshift_files = glob.glob('*/*/*/BD-shift_stats.txt')
len(BDshift_files)


Out[271]:
400

In [272]:
%%R -i BDshift_files

df_shift = list()
for(F in BDshift_files){
    tmp = read.delim(F, sep='\t') 
    FF = strsplit(F, '/') %>% unlist
    tmp$shared_perc = FF[1]
    tmp$perm_perc = FF[2]
    tmp$rep = FF[3]
    tmp$file = FF[4]
    df_shift[[F]] = tmp
}

df_shift = do.call(rbind, df_shift)
rownames(df_shift) = 1:nrow(df_shift)

df_shift = df_shift %>%
    filter(library %in% c(2,4,6)) %>%
    group_by(taxon, shared_perc, perm_perc, rep) %>%
    summarize(median = median(median)) %>%
    ungroup() %>%
    rename('median_true_BD_shift' = median)

df_shift %>% head(n=3)


Source: local data frame [3 x 5]

                           taxon shared_perc perm_perc   rep
                          (fctr)       (chr)     (chr) (chr)
1 Acaryochloris_marina_MBIC11017         100         0     1
2 Acaryochloris_marina_MBIC11017         100         0    10
3 Acaryochloris_marina_MBIC11017         100         0     2
Variables not shown: median_true_BD_shift (dbl)

In [273]:
%%R
df.j = inner_join(df_dBD, df_shift, c('taxon' = 'taxon',
                                       'shared_perc'='shared_perc',
                                       'perm_perc'='perm_perc',
                                       'rep'='rep')) %>%
    mutate(shared_perc = shared_perc %>% as.Num,
           perm_perc = perm_perc %>% as.Num) %>%
    filter(!is.na(delta_BD),
           perm_perc <= 20) %>%
    mutate(true_incorporator = ifelse(median_true_BD_shift > 0.002, TRUE, FALSE),
           true_atom_fraction_excess = median_true_BD_shift / 0.036,
           atom_fraction_excess = delta_BD / 0.036)
df.j %>% head(n=3)


                                 taxon mean_CM_control mean_CM_treatment
1       Acaryochloris_marina_MBIC11017        1.714107          1.711671
2 Acetobacter_pasteurianus_IFO_3283-03        1.717515          1.714771
3       Acetobacterium_woodii_DSM_1030        1.712993          1.702921
  stdev_CM_control stdev_CM_treatment     delta_BD shared_perc perm_perc rep
1      0.002785874       0.0042661210 -0.002435657          80        20   7
2      0.000842784       0.0019431914 -0.002743521          80        20   7
3      0.001254314       0.0009522154 -0.010071941          80        20   7
                        file median_true_BD_shift true_incorporator
1 OTU_abs1e9_PCR_sub_dBD.txt                    0             FALSE
2 OTU_abs1e9_PCR_sub_dBD.txt                    0             FALSE
3 OTU_abs1e9_PCR_sub_dBD.txt                    0             FALSE
  true_atom_fraction_excess atom_fraction_excess
1                         0          -0.06765715
2                         0          -0.07620891
3                         0          -0.27977613

In [274]:
%%R
# free memory
df_shift = df_dBD = NULL
# saving dataframe (if needed)
df.j.dBD = df.j

In [275]:
%%R -h 250 
# difference between true and estimated
df.j.dis.dBD = df.j %>%
    mutate(delta_excess = atom_fraction_excess * 100 - true_atom_fraction_excess * 100) %>%
    group_by(shared_perc, perm_perc, true_incorporator) %>%
    summarize(mean_delta_excess = mean(delta_excess),
              sd_delta_excess = sd(delta_excess)) %>%
    ungroup() %>%
    mutate(shared_perc = shared_perc %>% reorder(shared_perc %>% as.numeric),
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric)) %>%
    filter(true_incorporator == TRUE)
   
# plotting
ggplot(df.j.dis.dBD, aes(shared_perc, mean_delta_excess, 
                      color=perm_perc, group=perm_perc,
                      ymin=mean_delta_excess-sd_delta_excess,
                     ymax=mean_delta_excess+sd_delta_excess)) +
    geom_point() +
    geom_linerange(alpha=0.5, size=0.8) +
    geom_line() +
    scale_color_discrete('% taxa ranks\npermuted') +
    labs(x='% taxa shared', y='atom % excess\n(truth - estimate)') +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )


combined plot


In [276]:
%%R -w 670 -h 250
df.jj = rbind(df.j.dis.qSIP %>% mutate(method='qSIP'),
              df.j.dis.dBD %>% mutate(method='ΔBD'))

# plotting
p.comb = ggplot(df.jj, aes(shared_perc, mean_delta_excess, 
                      color=perm_perc, group=perm_perc,
                      ymin=mean_delta_excess-sd_delta_excess,
                     ymax=mean_delta_excess+sd_delta_excess)) +
    geom_point() +
    geom_linerange(alpha=0.5, size=0.5) +
    geom_line() +
    geom_line() +
    scale_color_discrete('% taxa ranks\npermuted') +
    labs(x='% taxa shared', y='13C atom % excess\n(truth - estimate)') +
    facet_grid(. ~ method) +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )
p.comb


qSIP+deltaBD


In [288]:
%%R -w 650 -h 250
df.jj = rbind(df.j.dis.qSIP %>% mutate(method='qSIP'),
              df.j.dis.dBD %>% mutate(method='Delta BD'))

# plotting
p.comb = ggplot(df.jj, aes(shared_perc, mean_delta_excess, 
                      color=perm_perc, group=perm_perc,
                      ymin=mean_delta_excess-sd_delta_excess,
                     ymax=mean_delta_excess+sd_delta_excess)) +
    geom_point() +
    geom_linerange(alpha=0.3, size=1) +
    geom_line() +
    scale_color_discrete('% taxa\nranks\npermuted') +
    labs(x='% taxa shared', y='13C atom % excess\n(truth - estimate)') +
    facet_grid(. ~ method) +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )
p.comb


Density plots


In [289]:
%%R
df.jj = rbind(df.j.qSIP %>% 
                  dplyr::select(atom_fraction_excess, true_atom_fraction_excess, shared_perc,
                                true_incorporator, perm_perc) %>%
                  mutate(method='qSIP'),
              df.j.dBD %>% 
                  dplyr::select(atom_fraction_excess, true_atom_fraction_excess, shared_perc,
                                true_incorporator, perm_perc) %>%
                  mutate(method='Delta BD')) %>%
    filter(perm_perc == '10',
           true_incorporator==TRUE,
           atom_fraction_excess!=0) %>%
    mutate(delta_excess = atom_fraction_excess * 100 - true_atom_fraction_excess * 100,
           shared_perc = shared_perc %>% reorder(shared_perc %>% as.numeric),
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric), 
           atom_perc_excess = atom_fraction_excess * 100,
           true_atom_perc_excess = true_atom_fraction_excess * 100) 

df.jj %>% head(n=3)


  atom_fraction_excess true_atom_fraction_excess shared_perc true_incorporator
1            0.6758098                         1          80              TRUE
2            0.7126439                         1          80              TRUE
3            0.6670081                         1          80              TRUE
  perm_perc method delta_excess atom_perc_excess true_atom_perc_excess
1        10   qSIP    -32.41902         67.58098                   100
2        10   qSIP    -28.73561         71.26439                   100
3        10   qSIP    -33.29919         66.70081                   100

In [290]:
%%R -w 700 -h 300

tmp = df.jj %>%
    dplyr::select(true_atom_perc_excess, shared_perc, method) %>%
    distinct(shared_perc, method) 

p.dens = ggplot(df.jj, aes(atom_perc_excess)) +
    geom_density(fill='grey70') +
    geom_vline(data=tmp, aes(xintercept=true_atom_perc_excess), linetype='dashed', alpha=0.5) +
    scale_x_continuous(limits=c(-20, 120)) +
    labs(x='13C atom % excess', y='Probability density') +
    facet_grid(method ~ shared_perc) +
    theme_bw() +
    theme(
        text = element_text(size=16)
    )
p.dens


Combined plot


In [291]:
%%R -w 700 -h 500

p.comb2 = cowplot::ggdraw() +
    geom_rect(aes(xmin=0, ymin=0, xmax=1, ymax=1), color='white', fill='white') +
    cowplot::draw_plot(p.comb, 0, 0.61, 1, 0.38) +
    cowplot::draw_plot(p.dens, 0, 0, 1, 0.60) +
    cowplot::draw_plot_label(c('A)', 'B)'), c(0, 0), c(1, 0.61))
p.comb2



In [292]:
%%R
outFile = 'microBetaDiv_qSIP-dBD.pdf'
ggsave(outFile, p.comb2, width=10, height=7.15)
cat('File written:', file.path(getwd(), outFile), '\n')


File written: /home/nick/notebook/SIPSim/dev/bac_genome1147/microBetaDiv/microBetaDiv_qSIP-dBD.pdf 

accuracy ~ pre-fractionation Bray-Curtis

  • mean Bray-Curtis among replicates

In [123]:
betaDiv_files = glob.glob('*/*/*/comm_betaDiv.txt')
len(betaDiv_files)


Out[123]:
400

In [127]:
%%R -i betaDiv_files

df_beta = list()
for(F in betaDiv_files){
    tmp = read.delim(F, sep='\t') 
    FF = strsplit(F, '/') %>% unlist
    tmp$shared_perc = FF[1]
    tmp$perm_perc = FF[2]
    tmp$rep = FF[3]
    df_beta[[F]] = tmp
}

df_beta = do.call(rbind, df_beta)
rownames(df_beta) = 1:nrow(df_beta)

# status
df_beta %>% head(n=3)


  library_x library_y      bray shared_perc perm_perc rep
1         1         2 0.5477131          80        20   7
2         1         3 0.4015788          80        20   7
3         1         4 0.4584426          80        20   7

In [180]:
%%R

# mean bray per parameter set
df_beta_s = df_beta %>%
    group_by(shared_perc, perm_perc, rep) %>%
    summarize(mean_BC = mean(bray)) %>%
    ungroup() %>%
    mutate(shared_perc = shared_perc %>% as.Num,
           perm_perc = perm_perc %>% as.Num, 
           rep = rep %>% as.Num)
df_beta_s %>% head(n=3)


Source: local data frame [3 x 4]

  shared_perc perm_perc   rep    mean_BC
        (dbl)     (dbl) (dbl)      (dbl)
1         100         0     1 0.08771624
2         100         0    10 0.12357141
3         100         0     2 0.20287997

In [181]:
%%R
# # joining to table on accuracy
# df_byClass.s.f %>% nrow %>% print
# df_byClass.s.f.j = inner_join(df_byClass.s.f %>%
#                                 mutate(shared_perc = shared_perc %>% as.Num,
#                                 perm_perc = perm_perc %>% as.Num),
#                             df_beta_s, 
#                             c('shared_perc'='shared_perc',
#                              'perm_perc'='perm_perc')) %>%
#     #filter(variables == 'Balanced Accuracy') %>%
#     mutate(perm_perc = perm_perc %>% as.character,
#            perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric),
#            shared_perc = shared_perc %>% as.character,
#            shared_perc = shared_perc %>% reorder(shared_perc %>% as.numeric))
    
# df_byClass.s.f.j %>% nrow %>% print
# df_byClass.s.f.j %>% head(n=3)


NULL

In [182]:
%%R
# confusion matrix data
vars = c('Balanced Accuracy', 'Sensitivity', 'Specificity')

df_byClass.f = df_byClass %>%
    filter(variables %in% vars) %>%
    mutate(shared_perc = shared_perc %>% as.Num,
           perm_perc = perm_perc %>% as.Num,
           rep = rep %>% as.Num)

df_byClass.f.j = inner_join(df_byClass.f, df_beta_s, 
                            c('shared_perc'='shared_perc',
                              'perm_perc'='perm_perc',
                              'rep'='rep')) %>%
    #filter(variables == 'Balanced Accuracy') %>%
    mutate(perm_perc = perm_perc %>% as.character,
           perm_perc = perm_perc %>% reorder(perm_perc %>% as.numeric),
           shared_perc = shared_perc %>% as.character,
           shared_perc = shared_perc %>% reorder(shared_perc %>% as.numeric))
    

df_byClass.f %>% head(n=3)


  library         variables    values shared_perc perm_perc rep
1       2       Sensitivity 0.5940594          80        20   7
2       2       Specificity 0.9168421          80        20   7
3       2 Balanced Accuracy 0.7554508          80        20   7
                     file method
1 DESeq2-cMtx_byClass.txt HR-SIP
2 DESeq2-cMtx_byClass.txt HR-SIP
3 DESeq2-cMtx_byClass.txt HR-SIP

In [259]:
%%R -w 700
# plotting
p.pnt = ggplot(df_byClass.f.j %>% filter(library==6), aes(mean_BC, values)) +
    stat_smooth(level=0.99, n=10) +
    labs(x='Mean Bray-Curtis distance among pre-fractionation communities') +
    facet_grid(variables ~ method, scale='free_y') +
    theme_bw() +
    theme(
        text = element_text(size=16),
        axis.title.y = element_blank()
    )
p.pnt


Notes

  • grey = 99% CI

In [260]:
%%R -i workDir
# saving the plot
F = file.path(workDir, 'microBetaDiv_Bray-smooth.pdf')
ggsave(F, p.pnt, width=10, height=7)
cat('File written:', F, '\n')


File written: /home/nick/notebook/SIPSim/dev/bac_genome1147//microBetaDiv_Bray-smooth.pdf 

Accuracy for certain scenarios


In [251]:
%%R
# calculating SD of 
df_byClass.f.j.s = df_byClass.f.j %>%
    filter(library==6) %>%
    group_by(variables, method, shared_perc) %>%
    summarize(mean_BC = first(mean_BC),
              mean_value = mean(values),
              sd_value = sd(values)) %>%
    ungroup() 

df_byClass.f.j.s %>% head


Source: local data frame [6 x 6]

          variables    method shared_perc   mean_BC mean_value   sd_value
             (fctr)    (fctr)      (fctr)     (dbl)      (dbl)      (dbl)
1 Balanced Accuracy Heavy-SIP          80 0.4566601  0.6688519 0.02343066
2 Balanced Accuracy Heavy-SIP          85 0.4495071  0.6742463 0.02987963
3 Balanced Accuracy Heavy-SIP          90 0.4613332  0.6862678 0.02290973
4 Balanced Accuracy Heavy-SIP          95 0.4693896  0.6891703 0.02653753
5 Balanced Accuracy Heavy-SIP         100 0.3678468  0.6995623 0.02301515
6 Balanced Accuracy    HR-SIP          80 0.4566601  0.7591432 0.03250748

In [254]:
%%R
df_byClass.f.j.s %>%
    filter(method=='MW-HR-SIP',
           variables=='Balanced Accuracy')


Source: local data frame [5 x 6]

          variables    method shared_perc   mean_BC mean_value   sd_value
             (fctr)    (fctr)      (fctr)     (dbl)      (dbl)      (dbl)
1 Balanced Accuracy MW-HR-SIP          80 0.4566601  0.8576313 0.02435424
2 Balanced Accuracy MW-HR-SIP          85 0.4495071  0.8686644 0.02273079
3 Balanced Accuracy MW-HR-SIP          90 0.4613332  0.8872503 0.01887123
4 Balanced Accuracy MW-HR-SIP          95 0.4693896  0.8967515 0.02371895
5 Balanced Accuracy MW-HR-SIP         100 0.3678468  0.9782744 0.01663307