In [9]:

    

from  os.path import basename
from  os.path import splitext
import os

pythonexe = "/usr/local/bin/python2.7"
qiimedir = "/opt/qiime-1.6.0/bin"
seqs   = "/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/merged/seqs.fna"
outdir = "/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime/"

identities = [ 0.9, 0.94, 0.97, 0.99 ]
refseqs = "/space/users/ams/data/midas/repseqs/97_V4.fasta"
taxfile = "/space/users/ams/data/midas/tax/midas_taxonomy.txt"
ncores = 10


    

In [10]:

    

for identity in identities:
    
    seqsfname = basename(seqs)
    stem = splitext(seqsfname)[0]
    outpath = "{0}/otus{1}".format(outdir, int(identity * 100) )
    print "writing to %s" % outpath
    
    !$pythonexe $qiimedir/pick_otus.py -i $seqs -o $outpath/uclust_picked_otus --otu_picking_method uclust --similarity $identity
    
    # label output with otu identity
    default_otusfname =  os.path.join(outpath, "uclust_picked_otus", stem + "_otus.txt")
    print default_otusfname
    otus = os.path.join(outpath, "uclust_picked_otus", "{0}_otus{1}.txt".format(stem, int(identity * 100)))
    print "otus: " + otus
    !mv $default_otusfname $otus
    repset = os.path.join(outpath, "{0}_rep_set{1}.fasta".format(stem, int(identity * 100)))
    replog = os.path.join(outpath, "{0}_rep_set{1}.log".format(stem, int(identity * 100)))
    
    !$pythonexe $qiimedir/pick_rep_set.py -i $otus -f $seqs -l $replog -o $repset --rep_set_picking_method most_abundant --sort_by otu
    print "midas taxonomy git hash:"
    githash = !git --git-dir=/space/users/ams/data/midas/.git --work-tree=/space/users/ams/data/midas/ log --pretty=oneline --abbrev-commit -n 1
    print githash
    
    !$pythonexe $qiimedir/parallel_assign_taxonomy_rdp.py -i $repset -o $outpath/rdp_assigned_taxonomy -T --jobs_to_start $ncores --seconds_to_sleep 60 --reference_seqs_fp $refseqs --id_to_taxonomy_fp $taxfile --confidence 0.8
    
    taxass = os.path.join(outpath, "rdp_assigned_taxonomy/{0}_rep_set{1}_tax_assignments.txt".format(stem, int(identity * 100)))
    biom_otutable = os.path.join(outpath, "{0}_{1}_otutable.biom".format(stem, int(identity * 100)))
    
    !$pythonexe $qiimedir/make_otu_table.py -i $otus -t $taxass -o $biom_otutable
    !$pythonexe $qiimedir/per_library_stats.py -i $biom_otutable


    

    




writing to /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus90
/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus90/uclust_picked_otus/seqs_otus.txt
otus: /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus90/uclust_picked_otus/seqs_otus90.txt
midas taxonomy git hash:
['28a4675 added incomplete reformat script for later...']
Num samples: 48
Num otus: 1362
Num observations (sequences): 2374197.0
Table density (fraction of non-zero values): 0.2367

Seqs/sample summary:
 Min: 2707.0
 Max: 80915.0
 Median: 56344.5
 Mean: 49462.4375
 Std. dev.: 20320.9805814
 Median Absolute Deviation: 10149.0
 Default even sampling depth in
  core_qiime_analyses.py (just a suggestion): 36011.0
 Sample Metadata Categories: None provided
 Observation Metadata Categories: taxonomy

Seqs/sample detail:
 AMPA126: 2707.0
 AMPA161: 9409.0
 AMPA232: 11906.0
 AMPA236: 14155.0
 AMPA160: 14834.0
 AMPA233: 15851.0
 AMPA162: 17762.0
 AMPA234: 18028.0
 AMPA235: 18407.0
 AMPA163: 19898.0
 AMPA164: 36011.0
 AMPA032: 42269.0
 AMPA034: 42984.0
 AMPA031: 46353.0
 AMPA030: 46409.0
 AMPA033: 46487.0
 AMPA727: 46527.0
 AMPA724: 46694.0
 AMPA725: 47456.0
 AMPA028: 48697.0
 AMPA029: 49408.0
 AMPA726: 52152.0
 AMPA044: 52360.0
 AMPA047: 55111.0
 AMPA103: 57578.0
 AMPA104: 58727.0
 AMPA105: 59020.0
 AMPA102: 59796.0
 AMPA106: 59808.0
 AMPA578: 62008.0
 AMPA579: 62283.0
 AMPA055: 62436.0
 AMPA056: 63327.0
 AMPA568: 63395.0
 AMPA054: 64305.0
 AMPA570: 65371.0
 AMPA566: 66129.0
 AMPA562: 66651.0
 AMPA053: 67114.0
 AMPA615: 67132.0
 AMPA564: 67249.0
 AMPA563: 67549.0
 AMPA580: 67724.0
 AMPA561: 68267.0
 AMPA057: 69226.0
 AMPA581: 71487.0
 AMPA632: 74825.0
 AMPA110: 80915.0
writing to /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus94
/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus94/uclust_picked_otus/seqs_otus.txt
otus: /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus94/uclust_picked_otus/seqs_otus94.txt
midas taxonomy git hash:
['28a4675 added incomplete reformat script for later...']
Num samples: 48
Num otus: 2541
Num observations (sequences): 2374197.0
Table density (fraction of non-zero values): 0.1999

Seqs/sample summary:
 Min: 2707.0
 Max: 80915.0
 Median: 56344.5
 Mean: 49462.4375
 Std. dev.: 20320.9805814
 Median Absolute Deviation: 10149.0
 Default even sampling depth in
  core_qiime_analyses.py (just a suggestion): 36011.0
 Sample Metadata Categories: None provided
 Observation Metadata Categories: taxonomy

Seqs/sample detail:
 AMPA126: 2707.0
 AMPA161: 9409.0
 AMPA232: 11906.0
 AMPA236: 14155.0
 AMPA160: 14834.0
 AMPA233: 15851.0
 AMPA162: 17762.0
 AMPA234: 18028.0
 AMPA235: 18407.0
 AMPA163: 19898.0
 AMPA164: 36011.0
 AMPA032: 42269.0
 AMPA034: 42984.0
 AMPA031: 46353.0
 AMPA030: 46409.0
 AMPA033: 46487.0
 AMPA727: 46527.0
 AMPA724: 46694.0
 AMPA725: 47456.0
 AMPA028: 48697.0
 AMPA029: 49408.0
 AMPA726: 52152.0
 AMPA044: 52360.0
 AMPA047: 55111.0
 AMPA103: 57578.0
 AMPA104: 58727.0
 AMPA105: 59020.0
 AMPA102: 59796.0
 AMPA106: 59808.0
 AMPA578: 62008.0
 AMPA579: 62283.0
 AMPA055: 62436.0
 AMPA056: 63327.0
 AMPA568: 63395.0
 AMPA054: 64305.0
 AMPA570: 65371.0
 AMPA566: 66129.0
 AMPA562: 66651.0
 AMPA053: 67114.0
 AMPA615: 67132.0
 AMPA564: 67249.0
 AMPA563: 67549.0
 AMPA580: 67724.0
 AMPA561: 68267.0
 AMPA057: 69226.0
 AMPA581: 71487.0
 AMPA632: 74825.0
 AMPA110: 80915.0
writing to /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus97
/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus97/uclust_picked_otus/seqs_otus.txt
otus: /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus97/uclust_picked_otus/seqs_otus97.txt
midas taxonomy git hash:
['28a4675 added incomplete reformat script for later...']
Num samples: 48
Num otus: 4586
Num observations (sequences): 2374197.0
Table density (fraction of non-zero values): 0.1619

Seqs/sample summary:
 Min: 2707.0
 Max: 80915.0
 Median: 56344.5
 Mean: 49462.4375
 Std. dev.: 20320.9805814
 Median Absolute Deviation: 10149.0
 Default even sampling depth in
  core_qiime_analyses.py (just a suggestion): 36011.0
 Sample Metadata Categories: None provided
 Observation Metadata Categories: taxonomy

Seqs/sample detail:
 AMPA126: 2707.0
 AMPA161: 9409.0
 AMPA232: 11906.0
 AMPA236: 14155.0
 AMPA160: 14834.0
 AMPA233: 15851.0
 AMPA162: 17762.0
 AMPA234: 18028.0
 AMPA235: 18407.0
 AMPA163: 19898.0
 AMPA164: 36011.0
 AMPA032: 42269.0
 AMPA034: 42984.0
 AMPA031: 46353.0
 AMPA030: 46409.0
 AMPA033: 46487.0
 AMPA727: 46527.0
 AMPA724: 46694.0
 AMPA725: 47456.0
 AMPA028: 48697.0
 AMPA029: 49408.0
 AMPA726: 52152.0
 AMPA044: 52360.0
 AMPA047: 55111.0
 AMPA103: 57578.0
 AMPA104: 58727.0
 AMPA105: 59020.0
 AMPA102: 59796.0
 AMPA106: 59808.0
 AMPA578: 62008.0
 AMPA579: 62283.0
 AMPA055: 62436.0
 AMPA056: 63327.0
 AMPA568: 63395.0
 AMPA054: 64305.0
 AMPA570: 65371.0
 AMPA566: 66129.0
 AMPA562: 66651.0
 AMPA053: 67114.0
 AMPA615: 67132.0
 AMPA564: 67249.0
 AMPA563: 67549.0
 AMPA580: 67724.0
 AMPA561: 68267.0
 AMPA057: 69226.0
 AMPA581: 71487.0
 AMPA632: 74825.0
 AMPA110: 80915.0
writing to /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus99
/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus99/uclust_picked_otus/seqs_otus.txt
otus: /space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime//otus99/uclust_picked_otus/seqs_otus99.txt
midas taxonomy git hash:
['28a4675 added incomplete reformat script for later...']
Num samples: 48
Num otus: 8276
Num observations (sequences): 2374197.0
Table density (fraction of non-zero values): 0.1262

Seqs/sample summary:
 Min: 2707.0
 Max: 80915.0
 Median: 56344.5
 Mean: 49462.4375
 Std. dev.: 20320.9805814
 Median Absolute Deviation: 10149.0
 Default even sampling depth in
  core_qiime_analyses.py (just a suggestion): 36011.0
 Sample Metadata Categories: None provided
 Observation Metadata Categories: taxonomy

Seqs/sample detail:
 AMPA126: 2707.0
 AMPA161: 9409.0
 AMPA232: 11906.0
 AMPA236: 14155.0
 AMPA160: 14834.0
 AMPA233: 15851.0
 AMPA162: 17762.0
 AMPA234: 18028.0
 AMPA235: 18407.0
 AMPA163: 19898.0
 AMPA164: 36011.0
 AMPA032: 42269.0
 AMPA034: 42984.0
 AMPA031: 46353.0
 AMPA030: 46409.0
 AMPA033: 46487.0
 AMPA727: 46527.0
 AMPA724: 46694.0
 AMPA725: 47456.0
 AMPA028: 48697.0
 AMPA029: 49408.0
 AMPA726: 52152.0
 AMPA044: 52360.0
 AMPA047: 55111.0
 AMPA103: 57578.0
 AMPA104: 58727.0
 AMPA105: 59020.0
 AMPA102: 59796.0
 AMPA106: 59808.0
 AMPA578: 62008.0
 AMPA579: 62283.0
 AMPA055: 62436.0
 AMPA056: 63327.0
 AMPA568: 63395.0
 AMPA054: 64305.0
 AMPA570: 65371.0
 AMPA566: 66129.0
 AMPA562: 66651.0
 AMPA053: 67114.0
 AMPA615: 67132.0
 AMPA564: 67249.0
 AMPA563: 67549.0
 AMPA580: 67724.0
 AMPA561: 68267.0
 AMPA057: 69226.0
 AMPA581: 71487.0
 AMPA632: 74825.0
 AMPA110: 80915.0

In [11]:

    

!pwd


    

    




/space/users/ams/Dropbox/docs/proj/core-paper/exp/EXP25-qualfilter-reduced_b/qiime

In [12]:

    

!ls


    

    




jobs  otus90  otus94  otus97  otus99  qiime_process.html  qiime_process.ipynb

In [12]: