In [1]:
%run ../../shared_setup.ipynb


docker image cggh/biipy:v1.6.0

In [2]:
tbl_dups = tbl_samples.duplicates(key=('cross', 'clone'))
tbl_dups.displayall()


0|cross 1|clone 2|sample 3|run 4|instrument 5|coverage 6|ID
3d7_hb3 C01 PG0065-C ERR019064 Illumina Genome Analyzer II 163X C01/PG0065-C/ERR019064
3d7_hb3 C01 PG0062-C ERR019070 Illumina Genome Analyzer II 108X C01/PG0062-C/ERR019070
3d7_hb3 C02 PG0055-C ERR019066 Illumina Genome Analyzer II 102X C02/PG0055-C/ERR019066
3d7_hb3 C02 PG0053-C ERR019067 Illumina Genome Analyzer II 73X C02/PG0053-C/ERR019067
3d7_hb3 C02 PG0056-C ERR019068 Illumina Genome Analyzer II 84X C02/PG0056-C/ERR019068
3d7_hb3 C02 PG0067-C ERR019073 Illumina Genome Analyzer II 126X C02/PG0067-C/ERR019073
7g8_gb4 AUD PG0112-C ERR029406 Illumina Genome Analyzer II 129X AUD/PG0112-C/ERR029406
7g8_gb4 AUD PG0112-CW ERR045639 Illumina HiSeq 2000 88X AUD/PG0112-CW/ERR045639
7g8_gb4 JC9 PG0111-C ERR029409 Illumina Genome Analyzer II 122X JC9/PG0111-C/ERR029409
7g8_gb4 JC9 PG0111-CW ERR045634 Illumina HiSeq 2000 121X JC9/PG0111-CW/ERR045634
7g8_gb4 JE11 PG0100-C ERR029404 Illumina Genome Analyzer II 134X JE11/PG0100-C/ERR029404
7g8_gb4 JE11 PG0100-CW ERR045630 Illumina HiSeq 2000 55X JE11/PG0100-CW/ERR045630
7g8_gb4 JF6 PG0079-C ERR027102 Illumina Genome Analyzer II 181X JF6/PG0079-C/ERR027102
7g8_gb4 JF6 PG0079-CW ERR045637 Illumina HiSeq 2000 94X JF6/PG0079-CW/ERR045637
7g8_gb4 KB8 PG0104-C ERR029148 Illumina Genome Analyzer II 116X KB8/PG0104-C/ERR029148
7g8_gb4 KB8 PG0104-CW ERR045642 Illumina HiSeq 2000 81X KB8/PG0104-CW/ERR045642
7g8_gb4 LA10 PG0086-C ERR029090 Illumina Genome Analyzer II 119X LA10/PG0086-C/ERR029090
7g8_gb4 LA10 PG0086-CW ERR045629 Illumina HiSeq 2000 66X LA10/PG0086-CW/ERR045629
7g8_gb4 NIC PG0095-C ERR027107 Illumina Genome Analyzer II 70X NIC/PG0095-C/ERR027107
7g8_gb4 NIC PG0095-CW ERR045631 Illumina HiSeq 2000 80X NIC/PG0095-CW/ERR045631
7g8_gb4 QF5 PG0078-C ERR029092 Illumina Genome Analyzer II 147X QF5/PG0078-C/ERR029092
7g8_gb4 QF5 PG0078-CW ERR045638 Illumina HiSeq 2000 82X QF5/PG0078-CW/ERR045638
7g8_gb4 XD8 PG0105-C ERR029144 Illumina Genome Analyzer II 121X XD8/PG0105-C/ERR029144
7g8_gb4 XD8 PG0105-CW ERR045628 Illumina HiSeq 2000 122X XD8/PG0105-CW/ERR045628
7g8_gb4 XF12 PG0102-C ERR029143 Illumina Genome Analyzer II 141X XF12/PG0102-C/ERR029143
7g8_gb4 XF12 PG0102-CW ERR045635 Illumina HiSeq 2000 96X XF12/PG0102-CW/ERR045635

In [3]:
tbl_dup_pairs = (
    tbl_dups
    .cut('cross', 'clone', 'ID')
    .join(
        tbl_dups.cut('clone', 'ID'), 
        key='clone'
    )
    .rename({2: 'ID1', 3: 'ID2'})
    .select(lambda row: row.ID1 != row.ID2)
    .addfield('pair', lambda row: sorted({row.ID1, row.ID2}))
    .cutout('ID1', 'ID2')
    .mergeduplicates(key='pair')
    .sort(key=('cross', 'clone', 'pair'))
)
tbl_dup_pairs.displayall()


0|pair 1|cross 2|clone
['C01/PG0062-C/ERR019070', 'C01/PG0065-C/ERR019064'] 3d7_hb3 C01
['C02/PG0053-C/ERR019067', 'C02/PG0055-C/ERR019066'] 3d7_hb3 C02
['C02/PG0053-C/ERR019067', 'C02/PG0056-C/ERR019068'] 3d7_hb3 C02
['C02/PG0053-C/ERR019067', 'C02/PG0067-C/ERR019073'] 3d7_hb3 C02
['C02/PG0055-C/ERR019066', 'C02/PG0056-C/ERR019068'] 3d7_hb3 C02
['C02/PG0055-C/ERR019066', 'C02/PG0067-C/ERR019073'] 3d7_hb3 C02
['C02/PG0056-C/ERR019068', 'C02/PG0067-C/ERR019073'] 3d7_hb3 C02
['AUD/PG0112-C/ERR029406', 'AUD/PG0112-CW/ERR045639'] 7g8_gb4 AUD
['JC9/PG0111-C/ERR029409', 'JC9/PG0111-CW/ERR045634'] 7g8_gb4 JC9
['JE11/PG0100-C/ERR029404', 'JE11/PG0100-CW/ERR045630'] 7g8_gb4 JE11
['JF6/PG0079-C/ERR027102', 'JF6/PG0079-CW/ERR045637'] 7g8_gb4 JF6
['KB8/PG0104-C/ERR029148', 'KB8/PG0104-CW/ERR045642'] 7g8_gb4 KB8
['LA10/PG0086-C/ERR029090', 'LA10/PG0086-CW/ERR045629'] 7g8_gb4 LA10
['NIC/PG0095-C/ERR027107', 'NIC/PG0095-CW/ERR045631'] 7g8_gb4 NIC
['QF5/PG0078-C/ERR029092', 'QF5/PG0078-CW/ERR045638'] 7g8_gb4 QF5
['XD8/PG0105-C/ERR029144', 'XD8/PG0105-CW/ERR045628'] 7g8_gb4 XD8
['XF12/PG0102-C/ERR029143', 'XF12/PG0102-CW/ERR045635'] 7g8_gb4 XF12

In [4]:
gatk = load_callsets(GATK_CALLSET_FN_TEMPLATE, 
                     crosses=('3d7_hb3', '7g8_gb4'),
                     variant_filter='FILTER_PASS',
                     call_filter=gatk_conf_calls)


2016-03-08 22:36:59.495438 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/3d7_hb3.gatk.final.npz
2016-03-08 22:37:00.476783 :: filter variants: excluding 155288 (80.9%) retaining 36598 (19.1%) of 191886 variants
2016-03-08 22:37:00.501638 :: filter calls: excluding 1413 (0.2%) retaining 767145 (99.8%) of 768558 calls
2016-03-08 22:37:00.502787 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/hb3_dd2.gatk.final.npz
2016-03-08 22:37:03.020434 :: filter variants: excluding 345466 (92.2%) retaining 29067 (7.8%) of 374533 variants
2016-03-08 22:37:03.050277 :: filter calls: excluding 14495 (1.3%) retaining 1060984 (98.7%) of 1075479 calls
2016-03-08 22:37:03.051810 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/7g8_gb4.gatk.final.npz
2016-03-08 22:37:05.685022 :: filter variants: excluding 358058 (92.8%) retaining 27709 (7.2%) of 385767 variants
2016-03-08 22:37:05.719416 :: filter calls: excluding 7569 (0.7%) retaining 1100791 (99.3%) of 1108360 calls

In [5]:
cortex = load_callsets(CORTEX_CALLSET_FN_TEMPLATE, 
                       crosses=('3d7_hb3', '7g8_gb4'),
                       variant_filter='FILTER_PASS',
                       call_filter=cortex_conf_calls)


2016-03-08 22:37:05.724627 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/3d7_hb3.cortex.final.npz
2016-03-08 22:37:05.958773 :: filter variants: excluding 29576 (52.0%) retaining 27296 (48.0%) of 56872 variants
2016-03-08 22:37:05.984022 :: filter calls: excluding 4849 (0.8%) retaining 568367 (99.2%) of 573216 calls
2016-03-08 22:37:05.985339 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/hb3_dd2.cortex.final.npz
2016-03-08 22:37:06.570209 :: filter variants: excluding 80272 (75.3%) retaining 26293 (24.7%) of 106565 variants
2016-03-08 22:37:06.593746 :: filter calls: excluding 38025 (3.9%) retaining 934816 (96.1%) of 972841 calls
2016-03-08 22:37:06.596495 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/7g8_gb4.cortex.final.npz
2016-03-08 22:37:07.223181 :: filter variants: excluding 85362 (78.5%) retaining 23394 (21.5%) of 108756 variants
2016-03-08 22:37:07.243406 :: filter calls: excluding 26666 (2.8%) retaining 909094 (97.2%) of 935760 calls

In [6]:
combined = load_callsets(COMBINED_CALLSET_FN_TEMPLATE, 
                         crosses=('3d7_hb3', '7g8_gb4'),
                         variant_filter='FILTER_PASS',
                         call_filter=combined_conf_calls)


2016-03-08 22:37:07.248474 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/3d7_hb3.combined.final.npz
2016-03-08 22:37:07.551897 :: filter variants: excluding 157 (0.4%) retaining 42087 (99.6%) of 42244 variants
2016-03-08 22:37:07.591250 :: filter calls: excluding 2439 (0.3%) retaining 881388 (99.7%) of 883827 calls
2016-03-08 22:37:07.592316 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/hb3_dd2.combined.final.npz
2016-03-08 22:37:07.936761 :: filter variants: excluding 450 (1.2%) retaining 36461 (98.8%) of 36911 variants
2016-03-08 22:37:07.986032 :: filter calls: excluding 31278 (2.3%) retaining 1317779 (97.7%) of 1349057 calls
2016-03-08 22:37:07.987848 :: loading /data/plasmodium/pfalciparum/pf-crosses/data/public/20141022/7g8_gb4.combined.final.npz
2016-03-08 22:37:08.339717 :: filter variants: excluding 304 (0.9%) retaining 34471 (99.1%) of 34775 variants
2016-03-08 22:37:08.391954 :: filter calls: excluding 20403 (1.5%) retaining 1358437 (98.5%) of 1378840 calls

In [7]:
def concordance(callsets, cross, pair):
    callset = callsets[cross]
    sample1, sample2 = pair
    g1 = callset['calldata'][sample1]['genotype']
    g2 = callset['calldata'][sample2]['genotype']
    called = (g1 >= 0) & (g2 >= 0)
    concord = called & (g1 == g2)
    discord = called & (g1 != g2)
    return nnz(concord), nnz(discord)

In [8]:
tbl_dup_pairs_discordance = (
    tbl_dup_pairs
    .addfield('gatk', lambda row: concordance(gatk, row.cross, row.pair))
    .addfield('cortex', lambda row: concordance(cortex, row.cross, row.pair))
    .addfield('combined', lambda row: concordance(combined, row.cross, row.pair))
    .convert(('gatk', 'cortex', 'combined'), lambda v: '%s/%s' % (v[1], v[0] + v[1]))
    .convert('cross', LABELS)
    .movefield('pair', 2)
    .convert('pair', lambda v: '%s vs %s' % tuple(v))
    .rename({'cross': 'Cross', 
             'clone': 'Clone', 
             'pair': 'Replicate pair',
             'gatk': 'Discordance (BWA/GATK callset)',
             'cortex': 'Discordance (Cortex callset)',
             'combined': 'Discordance (Combined callset)',})
)
tbl_dup_pairs_discordance.displayall(caption='duplicate discordance', index_header=False)


duplicate discordance
Cross Clone Replicate pair Discordance (BWA/GATK callset) Discordance (Cortex callset) Discordance (Combined callset)
3D7 x HB3 C01 C01/PG0062-C/ERR019070 vs C01/PG0065-C/ERR019064 3/36567 1/27152 3/42021
3D7 x HB3 C02 C02/PG0053-C/ERR019067 vs C02/PG0055-C/ERR019066 1/36551 0/27008 1/41977
3D7 x HB3 C02 C02/PG0053-C/ERR019067 vs C02/PG0056-C/ERR019068 1/36530 0/26943 1/41948
3D7 x HB3 C02 C02/PG0053-C/ERR019067 vs C02/PG0067-C/ERR019073 3/36569 0/27068 3/42010
3D7 x HB3 C02 C02/PG0055-C/ERR019066 vs C02/PG0056-C/ERR019068 2/36527 0/27022 2/41949
3D7 x HB3 C02 C02/PG0055-C/ERR019066 vs C02/PG0067-C/ERR019073 5/36573 1/27172 6/42029
3D7 x HB3 C02 C02/PG0056-C/ERR019068 vs C02/PG0067-C/ERR019073 1/36545 0/27090 1/41985
7G8 x GB4 AUD AUD/PG0112-C/ERR029406 vs AUD/PG0112-CW/ERR045639 32/27524 7/22423 15/33814
7G8 x GB4 JC9 JC9/PG0111-C/ERR029409 vs JC9/PG0111-CW/ERR045634 28/27556 8/22700 8/33998
7G8 x GB4 JE11 JE11/PG0100-C/ERR029404 vs JE11/PG0100-CW/ERR045630 30/27182 2/20800 9/32703
7G8 x GB4 JF6 JF6/PG0079-C/ERR027102 vs JF6/PG0079-CW/ERR045637 25/27529 8/22544 10/33878
7G8 x GB4 KB8 KB8/PG0104-C/ERR029148 vs KB8/PG0104-CW/ERR045642 25/27256 6/21939 13/33296
7G8 x GB4 LA10 LA10/PG0086-C/ERR029090 vs LA10/PG0086-CW/ERR045629 26/27393 2/21724 11/33365
7G8 x GB4 NIC NIC/PG0095-C/ERR027107 vs NIC/PG0095-CW/ERR045631 32/26991 3/19531 10/31909
7G8 x GB4 QF5 QF5/PG0078-C/ERR029092 vs QF5/PG0078-CW/ERR045638 34/27422 6/22349 18/33682
7G8 x GB4 XD8 XD8/PG0105-C/ERR029144 vs XD8/PG0105-CW/ERR045628 29/27562 13/22572 17/33917
7G8 x GB4 XF12 XF12/PG0102-C/ERR029143 vs XF12/PG0102-CW/ERR045635 32/27507 5/22459 18/33801