We will use distance between test segments computed in 140926-test-signal-jump to find sequence of segments that were likely together. Armed with this fact we can take the individual proababilities of each segment and combine it to form one probability that will be used to update the probabilites of all the segments in the sequence

the sequences are found using a greedy algoirthm that stops when a conflict is detected

the probabilities of segments should be combined by multiplying them, however this did not work well. Probably because the probabilites are not well calibrated. Taking the mean had a better effect.

Suppose you have a chain of segments: $i \in 1 \ldots N $

Each segment predicts a seizure $P_i$ or not $Q_i=1-P_i$

if a chain is negative then the probability is $\prod Q_i$ if a chain is positive the situation is more complex. There is a chance $U$ that a seizure detection even has happened and $V=1-U$ it did not. I estimate $U$ to be around $0.2$. So the probability is $\prod ( U * P_i + V*Q_i)$

or $\prod Q_i \times \Pi ( U \frac{P_i}{Q_i} +V )$

the ratio of positive to negative probability is just $r = \prod ( U \frac{P_i}{Q_i} +V )$ and probability is $1/(1+1/r)$



In [1]:

    
%matplotlib inline
from matplotlib import pylab as pl
import cPickle as pickle
import pandas as pd
import numpy as np
import os

individual segment probablility file



In [2]:

    
FNAME_IN = '../submissions/141022-predict.4.csv'

updated probability file



In [3]:

    
FNAME_OUT = '../submissions/141022-predict.6.csv'



In [4]:

    
!head {FNAME_IN}









    



clip,preictal
Dog_1_test_segment_0001.mat,0.4538743665290779
Dog_1_test_segment_0002.mat,0.1802596928014765
Dog_1_test_segment_0003.mat,0.21491236635118124
Dog_1_test_segment_0004.mat,0.22850784714844014
Dog_1_test_segment_0005.mat,0.2130232242761728
Dog_1_test_segment_0006.mat,0.2522360702569839
Dog_1_test_segment_0007.mat,0.16566075276892955
Dog_1_test_segment_0008.mat,0.28611734296043245
Dog_1_test_segment_0009.mat,0.1435247467328573



In [11]:

    
scores = pd.read_csv(FNAME_IN, index_col='clip', squeeze=True)
out_scores = scores.copy()



In [12]:

    
scores['Dog_2_test_segment_0004.mat']









    Out[12]:





0.26697910252110313



In [13]:

    
targets = list(set(['_'.join(f.split('_')[:2]) for f in scores.index.values]))
targets.sort()
targets









    Out[13]:





['Dog_1', 'Dog_2', 'Dog_3', 'Dog_4', 'Dog_5', 'Patient_1', 'Patient_2']



In [14]:

    
rows_updated = []
Ntotal = 0
for target in targets:
    print
    d = np.load('/Users/udi/Downloads/kaggle/seizure-prediction/distance/%s-test-jump-distance.npy'%target)
    N = d.shape[0]
    Ntotal += N
    print target, N
    dord = np.unravel_index(d.ravel().argsort(),d.shape)
    Nsequences = N/6
    
    # find good pairs of segments that are likely to be paired in time
    next_segment = [-1]*N
    previous_segment = [-1]*N
    for i,(s1,s2) in enumerate(np.array(dord).T):
        dist = d[s1,s2]
        if dist > -0.7: # from 140926-train-signal-jump
            break
        if next_segment[s1] != -1:
            #print i,'right conflict',dist
            continue
        if previous_segment[s2] != -1:
            #print i,'left conflict',dist
            continue
        c = 1
        j = s1
        while previous_segment[j] != -1:
            j = previous_segment[j]
            c += 1
        j = s2
        c += 1
        while next_segment[j] != -1:
            j = next_segment[j]
            c += 1
        if c > 6:
            continue
        next_segment[s1] = s2
        previous_segment[s2] = s1
#     if i < Nsequences:
#         print 'skip'
#         continue
    # check code
    for i in range(N):
        if next_segment[i] != -1:
            assert previous_segment[next_segment[i]] == i

    # find good sequences
    sequences = []
    for i in range(N):
        if previous_segment[i] == -1 and next_segment[i] != -1:
            j = i
            sequence = [j]
            while next_segment[j] != -1:
                j = next_segment[j]
                sequence.append(j)
            sequences.append(sequence)
    len_sequences = [len(sequence) for sequence in sequences]
    print '#sequences',len(sequences), '%segments that was sequenced',sum(len_sequences)/float(N), 'longest sequence', max(len_sequences)
    print sequences

    #compute probability for sequences
    sequences_prb = []
    for sequence in sequences:
        p0 = 1.
        q0 = 1.
        p1 =0.
        p2 = 0.
        p3 = 1.
        U = 0.2 # chance of seizure detection event in a preictal segment
        V = 1-U
        for s in sequence:
            P = scores['%s_test_segment_%04d.mat'%(target,s+1)]
            Q = 1.-P
            p0 *= P
            q0 *= Q
            p1 += P
            if P > p2:
                p2 = P
            p3 *= (U * P/Q + (1-U))
        p0 = p0 / (p0+q0)
        p1 = p1 / len(sequence)
        p2 = p2
        p3 = 1./(1+1./p3)
#         print p0, p1, p2, p3
        sequences_prb.append(p2)
    # fix probability for segments in sequences
    for p,sequence in zip(sequences_prb,sequences):
        # all segments in the same sequence will be assigned the same probability
        for s in sequence:
            row_name = '%s_test_segment_%04d.mat'%(target,s+1)
            rows_updated.append(row_name)
            out_scores[row_name] = p









    



Dog_1 502
#sequences 101 %segments that was sequenced 0.972111553785 longest sequence 6
[[3, 493, 475, 178], [10, 177, 271, 255, 60], [13, 231, 159], [14, 317, 276, 15, 123], [16, 453, 425, 285, 191], [21, 32, 236, 199, 218], [22, 291, 381, 92, 268], [28, 124, 88, 215, 197, 242], [29, 398, 12, 224, 31, 0], [33, 390, 383, 309, 119], [40, 256, 262, 361, 428], [41, 357, 339, 153, 174, 364], [47, 211, 117, 479, 185, 246], [53, 126], [56, 24, 138], [68, 406, 282, 161, 468, 427], [71, 382, 192, 190, 463, 328], [72, 323, 351, 496, 385, 275], [73, 384, 444], [74, 290], [76, 127, 146, 106, 389, 349], [77, 204, 347, 214, 380, 462], [78, 375, 227, 109, 202, 405], [79, 48, 244, 114], [93, 321, 186, 473, 210, 298], [95, 452, 407, 445, 115, 358], [103, 171, 252, 175, 412, 154], [110, 85, 7, 258], [128, 373, 477, 356, 50, 500], [133, 288, 260, 490, 331, 424], [137, 162, 287, 129, 344], [139, 18, 313, 65], [149, 116, 97, 107, 219, 284], [150, 148, 112, 253, 283], [152, 179, 194, 377, 99, 87], [158, 140, 411, 143, 352, 474], [160, 434, 267, 293, 442, 221], [169, 302, 343, 232], [184, 173, 485, 96, 156, 480], [205, 187, 207, 82, 472], [216, 155], [225, 397, 228, 448, 35, 318], [226, 66, 134, 476, 44, 108], [234, 329, 438, 147, 136], [235, 37, 84, 336, 295], [237, 426, 118], [238, 492, 58, 371, 222], [247, 223, 249, 102, 62, 42], [251, 266, 483, 125, 61], [257, 122, 326, 432, 332], [261, 248, 416, 132, 70, 495], [265, 144], [269, 327, 270, 52, 8, 98], [274, 100], [278, 36, 198, 387, 69, 11], [289, 470, 330], [294, 341, 176, 245, 142, 292], [299, 478, 90, 353], [301, 366, 342, 30], [303, 19], [304, 230, 410, 471, 286, 201], [306, 421, 1], [308, 121, 404], [311, 212], [315, 163, 131, 80], [320, 57, 413, 75, 196, 27], [325, 436, 165, 279], [340, 281, 229, 333, 217, 233], [348, 213, 360, 441, 4, 220], [354, 181, 431, 338, 200], [355, 83, 39], [359, 409, 491], [365, 277, 322, 130, 2, 378], [367, 484, 180, 335, 296, 486], [369, 439, 488, 362, 312, 422], [370, 316, 300, 120, 141, 400], [374, 408, 447, 104, 297, 429], [379, 443, 449, 38, 399, 417], [388, 113, 280, 437, 168], [391, 94, 337], [393, 145, 350, 319, 324, 334], [401, 250, 9, 172], [414, 459, 45, 363], [415, 273], [418, 208, 239, 67, 345], [419, 195, 5, 420], [423, 465, 151, 386], [450, 395, 455, 34, 307, 376], [451, 17, 454, 314, 254, 433], [457, 460, 310, 456, 372], [461, 193, 91, 167, 188, 259], [464, 135, 89, 264, 368], [466, 209, 263, 346, 26, 170], [467, 243, 101, 54, 240, 394], [469, 51, 497, 43, 20], [481, 489, 430], [482, 241, 440, 49, 111, 59], [487, 206, 25, 157, 435, 166], [494, 63, 189, 458], [498, 305, 403], [499, 446, 182, 392, 164]]

Dog_2 1000
#sequences 201 %segments that was sequenced 0.975 longest sequence 6
[[11, 336, 161, 793, 310], [13, 179, 925, 636, 444], [16, 862, 329, 618], [35, 175, 472, 152, 340, 606], [36, 480, 880, 304, 421, 569], [44, 346, 660, 383, 626, 21], [49, 902, 944, 490, 488, 337], [56, 559, 980, 287, 414, 464], [59, 234, 352, 820, 14, 42], [61, 297, 45, 504, 760, 630], [62, 859, 858, 785, 133, 30], [65, 755, 465, 459, 550, 871], [69, 68, 351, 830], [74, 302, 775, 588, 673, 184], [80, 776, 108, 293, 9, 47], [92, 306, 455, 101, 994, 331], [93, 598, 695, 699, 638, 33], [94, 973], [97, 907, 142], [98, 223], [99, 359, 326, 879, 813, 333], [104, 889, 798, 733, 50, 72], [106, 217, 250, 892, 702, 555], [109, 63, 404, 447, 365, 763], [114, 79, 87, 216, 469, 977], [115, 57, 979, 594, 185, 341], [129, 722, 116, 640], [141, 620, 573, 51, 930], [146, 634, 281, 498, 535, 674], [148, 762], [158, 198, 658, 652, 566, 113], [166, 105, 371, 197, 338], [169, 257, 73, 380], [173, 954, 761], [176, 662, 897, 117, 987, 539], [193, 38], [205, 322, 233, 292], [209, 509], [214, 419, 357, 221, 764, 252], [225, 384, 582, 251, 289, 616], [230, 402, 121, 267, 347, 46], [236, 783, 291], [237, 857, 515, 969, 266, 532], [238, 578, 112, 400, 139], [249, 202, 600, 277, 196, 296], [256, 393, 2, 220, 482, 523], [258, 531, 201, 965, 835, 385], [260, 593, 22, 424, 312, 160], [270, 100, 507, 589, 602, 525], [284, 663, 716, 282, 826, 833], [294, 305, 283, 917, 136, 510], [295, 332, 466, 24, 558, 853], [299, 685, 831, 334, 298, 452], [303, 194, 877, 922, 984, 715], [313, 408, 707, 752], [319, 407, 819], [335, 467, 911, 103, 745, 561], [348, 240, 855, 836, 806, 503], [358, 164, 157, 731, 309, 723], [360, 413, 491], [363, 711, 328, 586, 991], [370, 608, 909], [374, 130, 10, 572, 60, 495], [376, 215, 789, 948, 327, 438], [394, 703, 19, 330, 269], [399, 259], [403, 95, 131, 941, 955, 574], [406, 748, 318, 533, 89, 111], [415, 841, 832, 255, 612, 581], [416, 76, 502, 777], [417, 982, 300, 700, 665, 538], [420, 725, 369, 570, 924, 373], [425, 643, 647], [432, 307, 445, 474, 471], [433, 200, 271, 617, 568, 468], [434, 720, 521, 993], [436, 840, 856, 219, 536], [437, 245, 546, 514, 657, 323], [440, 839], [449, 458, 661, 349], [450, 83, 920, 794, 751], [451, 58, 315, 772, 124, 91], [456, 874, 517, 943, 637, 957], [461, 423, 769, 321, 784], [475, 787], [484, 392], [486, 885, 805, 27, 378], [494, 147, 54, 850, 560, 929], [496, 992], [497, 900, 844, 738, 246, 625], [501, 278, 686, 718, 649, 726], [512, 401, 239, 122, 183, 967], [518, 659], [522, 809, 680], [528, 189, 460, 981, 155], [530, 778], [534, 477], [537, 995, 77, 442], [547, 366], [548, 6, 386], [551, 629, 31, 614, 361, 192], [553, 64, 435, 818, 165], [554, 153], [563, 224, 162, 213, 5], [579, 144, 852, 986], [584, 549, 916, 285], [585, 261, 43, 754, 441, 687], [591, 903, 187, 48, 552, 542], [601, 262, 653, 688, 747], [603, 963, 757, 0], [610, 621], [628, 500, 997, 811, 485], [632, 623, 691, 908, 741, 156], [633, 273, 807, 150], [642, 126], [644, 375, 190, 427, 481, 120], [646, 195], [648, 453, 431, 824, 82, 132], [650, 983], [654, 186, 487, 395, 272], [655, 631, 125], [656, 627], [664, 232, 792], [666, 177, 127, 998, 170, 32], [667, 950, 443, 354], [671, 694, 67, 254], [678, 635, 208, 286, 575, 682], [681, 41, 167, 896, 780], [689, 492, 975, 364, 211, 96], [693, 705, 590, 15, 571, 599], [697, 339, 846, 937], [706, 692, 556, 607], [708, 3, 226, 140, 838, 301], [709, 290, 849, 88, 947, 827], [710, 624], [713, 342, 730, 397, 749, 834], [717, 968, 324, 675, 288], [719, 345, 454, 412, 765, 188], [721, 123, 280, 439, 817], [727, 25, 446, 520, 936], [735, 390, 802, 398, 526], [736, 696, 381, 473, 949, 279], [739, 734, 970, 866, 85, 684], [742, 428, 23, 956, 935, 457], [743, 499, 479, 904, 609], [756, 753, 379, 567, 75, 53], [758, 462, 174, 163, 893, 516], [759, 84, 235, 118], [766, 714, 418, 815, 316, 851], [768, 396, 808], [771, 932, 972, 191, 138, 524], [786, 888, 128, 732, 906, 669], [788, 367, 86, 387, 641, 781], [790, 677, 26, 37, 276, 355], [795, 690, 344, 946, 847, 698], [797, 222, 990, 203, 275, 962], [799, 430, 910, 181, 263, 145], [803, 362, 562, 767, 508, 541], [804, 576, 182, 886, 28, 489], [812, 382, 529, 274, 248, 227], [814, 878, 988, 864, 527, 800], [821, 429, 887], [822, 40, 611, 210], [823, 729, 343, 583, 107, 905], [828, 180, 20, 923, 228], [829, 895, 311, 178], [837, 953, 356], [842, 921, 770, 377, 672, 476], [845, 505, 746, 172, 724, 320], [848, 372, 592, 872, 206, 71], [854, 934, 231, 704], [861, 989, 513, 883, 587, 154], [867, 7, 595, 791, 773], [869, 264, 317, 939, 645, 478], [870, 212], [873, 564, 959], [876, 448, 619, 810, 18, 974], [881, 668, 483], [884, 898, 137, 615, 353, 410], [891, 171, 391, 890, 580, 927], [894, 12, 875, 999, 90, 265], [899, 368, 110, 779, 389, 241], [901, 701, 8, 914], [912, 151, 825, 543], [913, 557], [915, 134], [918, 966, 39, 463, 960, 613], [919, 52, 737, 268, 728], [926, 961, 4, 774, 816, 868], [928, 409, 218, 622, 796, 744], [931, 34, 143, 605, 253, 405], [940, 244, 670, 740, 519], [942, 683], [945, 676, 243, 411, 750, 511], [951, 493, 1, 565, 976, 314], [952, 540, 679, 119, 325, 545], [958, 506, 199, 426, 168], [964, 78, 102, 865, 604, 247], [971, 863], [985, 29, 801, 66, 938, 17], [996, 882, 782, 470]]

Dog_3 907
#sequences 178 %segments that was sequenced 0.943770672547 longest sequence 6
[[3, 81, 539, 270, 702, 582], [13, 226, 111, 33, 645], [14, 116, 537, 654], [22, 211, 854, 889, 176, 486], [32, 396, 59, 569, 594], [36, 714, 218, 630], [44, 456, 906, 820, 521, 538], [45, 453, 480, 215, 688, 150], [48, 728, 665, 363, 802, 762], [56, 302, 766], [63, 432, 669, 664, 739, 483], [67, 708, 241, 715], [70, 368, 672, 600, 96], [76, 450, 704, 120], [78, 513, 187], [83, 170], [85, 497, 308, 718, 5, 751], [87, 847, 661, 720, 519, 327], [88, 16, 238, 581], [95, 19, 818, 458], [97, 49, 850], [101, 230], [107, 851, 246, 683, 596, 485], [118, 72, 635, 481, 874, 89], [121, 34], [131, 599, 39, 434, 367, 52], [135, 603, 852, 873, 780, 243], [137, 656, 564, 696], [138, 348, 168, 359], [151, 615, 415, 400, 423, 748], [152, 40, 154, 839, 98, 756], [159, 210, 648, 698, 534, 791], [162, 863, 202, 307, 546, 742], [164, 427, 279, 160], [166, 901, 468, 487, 245, 342], [169, 779, 42, 588, 53, 275], [172, 355], [174, 82, 283, 188, 310, 811], [179, 303, 724, 489], [184, 35, 445, 580, 366, 51], [192, 631, 551, 12, 345, 786], [193, 145, 837], [197, 832, 626, 104, 325, 277], [199, 147, 2, 171, 182, 347], [201, 372, 565, 893, 710, 828], [209, 177, 770, 776, 231], [224, 848, 727, 690, 55, 821], [227, 808, 524, 11, 760, 701], [228, 394, 280, 31, 709], [235, 625, 540, 252, 619, 735], [244, 161], [255, 774, 681, 234, 767, 21], [256, 122, 510, 130, 27, 50], [259, 102, 526, 764, 382, 872], [263, 389, 498, 110, 117, 207], [264, 433, 844, 757, 262, 655], [267, 650, 562, 299, 712, 827], [269, 541, 807, 864, 790], [271, 298, 38, 428, 388, 426], [273, 508, 721, 425, 717, 379], [286, 555, 680, 525, 41], [289, 608, 689, 148, 447, 272], [293, 733, 198, 297, 418, 319], [300, 612, 305, 23], [306, 897, 361, 265], [313, 499, 18, 667, 755, 758], [329, 572, 217, 268, 203, 511], [331, 191, 260, 501, 369, 552], [338, 205, 559, 406, 333, 804], [339, 20, 143, 459, 846, 311], [343, 896, 741, 402, 640, 216], [346, 576, 587, 662], [354, 413], [357, 707, 636, 583], [358, 92, 431, 890, 105, 813], [362, 841, 798, 561, 810, 189], [365, 532], [371, 376, 185, 250, 886, 77], [373, 606, 208, 787, 782, 71], [374, 115, 341, 639, 793], [378, 644, 797], [380, 417], [386, 557, 765, 274, 647, 287], [393, 407, 146, 693], [397, 505, 318, 337], [409, 805, 421, 530, 10, 660], [411, 697, 899, 183, 340, 642], [412, 575, 290, 531, 401, 477], [440, 25, 732, 212, 866, 336], [443, 479, 200, 678, 233, 180], [444, 220, 91, 429, 157], [454, 119], [457, 106], [461, 291], [462, 370, 527], [466, 424, 139, 194, 549], [473, 254], [474, 822], [478, 883, 258, 550, 703, 57], [490, 853, 577], [492, 834, 28, 613, 836, 422], [500, 682, 628, 123], [504, 620, 126, 882, 384], [509, 74, 610, 520, 623, 391], [512, 436, 403, 633, 186, 364], [514, 24], [517, 206, 114, 452, 794, 439], [522, 598, 37, 618, 54], [535, 892, 885, 545, 591], [542, 178, 0, 796, 304], [543, 190, 229, 590, 112, 410], [544, 536, 506], [556, 257, 460, 488, 46, 395], [558, 240], [567, 455, 643, 515, 141, 566], [579, 113], [585, 223, 125, 496], [602, 745, 288, 595, 142], [604, 140, 795, 548, 586, 62], [614, 716, 158, 93, 888, 824], [616, 759, 204, 833, 442, 8], [617, 632, 788, 43, 624, 484], [622, 387, 695, 840, 686], [627, 773, 266, 589, 856, 747], [634, 219, 295, 335, 446, 181], [641, 349, 451, 322, 247], [653, 7, 420, 173, 73, 865], [657, 467, 66], [658, 878, 475, 404, 578, 278], [666, 153, 605], [668, 700, 502, 312], [670, 323], [673, 465, 109, 242, 838, 849], [685, 213, 175, 675, 674, 377], [691, 875, 330, 405, 124], [694, 528], [705, 692, 533, 4, 734, 292], [706, 570, 470], [719, 144, 740, 746, 884, 90], [722, 713, 86, 155], [731, 469, 871], [736, 597, 601, 495, 753, 285], [738, 769, 103, 136, 659, 6], [743, 30, 563, 237, 408, 754], [744, 777, 99, 638, 471, 284], [749, 857, 571, 503, 895, 568], [752, 464, 789, 414, 353], [761, 523], [771, 768, 232, 225], [778, 60, 281, 554, 801, 221], [781, 507, 493, 763, 1, 573], [784, 441, 249], [785, 356, 726], [799, 862, 806, 845, 646], [800, 398, 317, 320, 699], [803, 817, 687, 449, 261, 195], [814, 47, 870, 609, 58, 887], [816, 792, 783, 435, 61, 352], [819, 843, 671, 239, 128, 29], [823, 869, 592], [825, 903], [826, 314, 64, 775], [829, 472, 399, 236, 812, 416], [830, 607, 711, 165, 127, 560], [831, 9, 842, 438], [855, 84, 430, 463, 652, 69], [858, 360, 65], [859, 494, 276, 79, 381, 332], [860, 679, 17, 684], [861, 156], [867, 737, 880, 133, 676, 809], [876, 375, 344, 94, 132], [877, 547, 316, 621, 491, 725], [879, 80, 553, 196, 437, 894], [891, 129, 750, 529], [902, 108, 482, 723, 593, 419], [904, 448], [905, 214]]

Dog_4 990
#sequences 172 %segments that was sequenced 0.987878787879 longest sequence 6
[[0, 399, 297, 940, 455, 284], [1, 260, 101, 478, 415, 236], [7, 759, 635, 264, 216, 848], [12, 184, 479, 829, 139, 724], [14, 581, 429, 714, 660, 582], [15, 24, 960, 519, 931, 965], [25, 315, 527, 323, 595, 133], [31, 328, 740, 637, 167, 942], [33, 631, 493, 561, 421, 74], [36, 964, 75, 141, 790, 775], [37, 340, 532, 632, 883, 518], [38, 978, 678], [40, 387, 810, 690, 930, 679], [44, 875, 116, 405, 118, 565], [46, 402, 971, 446, 506, 901], [55, 170, 319, 767, 925, 252], [56, 431, 566, 815, 124, 296], [60, 127, 847, 634, 778, 285], [63, 528, 362, 359, 346, 332], [67, 937, 987, 148, 948, 72], [85, 959, 897, 388, 962, 580], [87, 584, 902, 99, 289, 50], [92, 643, 212, 240, 335, 501], [93, 80], [103, 737, 721, 709, 131, 440], [112, 827, 155, 106, 890, 244], [120, 122, 281, 727, 406, 820], [123, 383, 809, 970, 82, 333], [125, 647, 374, 624, 312, 730], [128, 134], [136, 853, 417, 230, 494, 966], [138, 235, 613, 208, 349, 413], [140, 499, 604, 696, 379, 720], [142, 341, 860, 409, 655, 791], [159, 628, 261, 921, 375, 586], [163, 817, 648, 540, 39, 626], [172, 680, 295, 68, 214, 194], [175, 976, 16, 688, 924, 262], [176, 743, 196, 972, 702, 173], [177, 795, 520, 713, 439, 433], [197, 545, 531, 17, 756, 674], [204, 749, 876, 471, 114, 866], [211, 638, 376, 102, 373, 259], [220, 502, 770, 908, 985, 828], [225, 813, 344, 907, 465, 652], [242, 881, 207, 878, 253, 956], [256, 516, 473, 983, 597, 485], [263, 288, 186, 22, 444, 859], [276, 291, 313, 662, 272, 305], [279, 222, 327, 448, 729, 504], [282, 957, 497, 269, 656, 426], [294, 156, 357, 733, 698, 891], [299, 152, 183, 456, 861, 880], [300, 239, 926, 851, 392, 474], [303, 11, 10, 191, 89, 364], [308, 54], [309, 361, 187, 585, 728, 154], [310, 377, 783, 830, 672, 412], [314, 974, 542, 741, 832, 164], [320, 507, 824, 895, 715, 378], [329, 685, 523, 673, 686, 543], [330, 98, 144, 747, 988, 554], [338, 768, 703, 927, 511, 893], [343, 951, 943, 203, 760, 608], [350, 766, 835, 807, 837, 365], [353, 107, 664, 546, 18, 571], [372, 165, 923, 961, 66, 464], [389, 355, 337, 805, 583, 569], [390, 578, 599, 706, 910, 742], [394, 51, 564, 739, 316, 487], [395, 110, 419, 606, 977, 217], [396, 984, 257, 658, 954, 663], [403, 162, 659, 754, 798, 53], [404, 846, 58, 318, 229, 731], [411, 401, 286, 514, 398, 657], [424, 900, 251, 618, 232, 691], [425, 722, 258], [430, 894, 711, 23, 45, 462], [434, 27, 748, 787, 290, 933], [436, 734, 842, 348, 562, 416], [441, 619, 121, 573, 324, 544], [443, 794, 843, 77, 234, 808], [445, 633], [451, 449, 935, 873, 19, 592], [458, 687, 950, 629, 917, 854], [461, 906, 489, 322, 292, 552], [469, 796, 146, 226, 147, 200], [472, 958, 834, 761, 968, 109], [481, 877, 826, 973, 969, 247], [490, 919, 354, 90, 625, 548], [492, 188, 2, 505, 306, 3], [496, 273, 932, 73, 418], [498, 755, 718, 453, 168, 223], [500, 301, 233, 231, 459, 612], [509, 423, 13, 321, 132, 889], [513, 630, 366, 454, 428, 438], [515, 547, 605, 553, 52, 683], [522, 182, 29], [524, 850, 670, 797, 525, 945], [535, 278, 475, 677, 205, 601], [536, 836, 869, 115, 503, 166], [537, 248, 572, 905, 886, 477], [551, 28, 8, 589, 725, 801], [555, 920, 967, 654, 468, 470], [558, 484, 266, 979, 181, 852], [559, 903, 675, 936, 97, 145], [570, 385, 726, 526, 250], [575, 151, 71, 574, 283, 717], [577, 81, 653, 550, 704, 482], [588, 530, 407, 339, 457, 363], [596, 694, 735, 35, 466, 275], [598, 447, 645], [602, 265, 255, 224, 47, 195], [610, 667, 594, 342, 360, 169], [614, 137, 135, 311, 157, 111], [615, 369, 228, 892, 784, 368], [616, 517, 408, 215, 849, 280], [620, 178, 567, 161, 896, 76], [623, 975, 627, 593, 331, 947], [636, 867, 946, 707, 84, 367], [641, 65, 888, 644, 757, 336], [642, 96, 665, 414, 270, 317], [651, 668, 736, 833, 510, 149], [661, 61], [684, 914, 79, 126, 249], [692, 83, 245, 695, 746, 541], [697, 871, 100], [699, 622, 666, 307, 753, 862], [710, 386, 708, 911, 856, 792], [712, 671], [719, 928, 435, 922, 237, 804], [723, 206, 639, 752, 776, 20], [738, 48, 70, 483, 840, 986], [745, 351, 391, 4, 774], [750, 982, 352, 744, 108, 590], [751, 287, 274, 5, 370, 432], [762, 865, 870, 302, 189, 855], [763, 193, 885, 676, 649, 32], [764, 86, 789, 213, 872, 913], [765, 646, 650, 158, 427], [771, 944, 909, 380, 160, 788], [777, 912, 486, 825, 129, 915], [779, 732, 64, 603, 591, 803], [786, 533, 410, 939, 617, 839], [806, 780, 549, 609, 293, 452], [811, 938, 347, 450, 221, 521], [812, 326, 219, 989, 62, 420], [814, 802, 793, 700, 772, 201], [816, 400, 246, 277, 192, 334], [818, 304, 397, 874, 693, 238], [821, 6, 104, 105, 382], [822, 781, 705, 467, 701], [823, 480, 180, 199, 57, 841], [831, 463, 539, 952, 785, 91], [838, 576, 579, 916, 568, 9], [844, 393, 113, 78, 356, 782], [845, 981, 254, 884, 381, 198], [858, 202], [863, 460, 21, 799, 179, 495], [864, 512, 43, 882, 267, 769], [868, 185, 30, 560, 209, 143], [879, 227, 358, 773, 669, 689], [887, 819, 563, 26, 243, 325], [899, 556, 174, 508, 488, 529], [904, 963, 117, 587, 600, 69], [918, 268, 442, 934, 171, 241], [929, 476, 534, 218, 94, 59], [941, 758, 345, 491, 800, 681], [949, 150, 153, 88], [953, 42, 298, 857, 607, 41], [955, 621, 611, 190, 682, 49], [980, 422, 716, 130]]

Dog_5 191
#sequences 36 %segments that was sequenced 0.905759162304 longest sequence 6
[[1, 131, 105, 139, 147, 154], [3, 59, 30, 121], [11, 100, 79, 119, 55, 2], [12, 184], [17, 155, 28, 125, 98, 185], [18, 63, 137], [29, 122], [34, 49, 27, 25, 165], [38, 84, 107], [41, 21], [50, 179, 187, 57, 133, 126], [56, 170, 13, 58, 144, 14], [60, 80, 96, 15, 152, 5], [65, 111], [67, 108, 52, 90, 159], [70, 37, 64], [71, 35, 22, 112], [77, 113, 114, 135], [86, 82, 157, 24, 32, 74], [87, 183], [94, 161, 171, 46, 176, 39], [99, 43, 129, 75, 178, 88], [104, 162, 118, 66, 97, 76], [124, 93, 148, 136, 169, 132], [128, 48, 42, 160, 0, 73], [134, 85, 78, 158, 102, 189], [138, 92, 109, 44, 10, 7], [141, 95, 91, 115, 117, 163], [145, 130, 103, 123, 120, 180], [146, 149, 6, 36, 140], [153, 40, 150, 164, 142, 101], [166, 182, 33, 23, 190, 106], [167, 127, 69, 188], [168, 173, 151], [175, 81, 172, 186, 83, 20], [177, 26, 143, 68, 62, 51]]

Patient_1 195
#sequences 44 %segments that was sequenced 0.738461538462 longest sequence 6
[[1, 104, 158], [3, 54, 27], [10, 118, 25, 83], [21, 112], [22, 170], [26, 44, 65], [30, 157, 92], [32, 193], [36, 61], [42, 194, 64], [43, 148, 46, 35, 79], [49, 129, 37, 48, 23, 154], [56, 66, 53, 87], [67, 47], [68, 51], [73, 159, 156, 183, 160], [76, 171], [78, 77, 147, 172], [82, 121, 4, 12], [86, 15], [89, 110, 117, 13, 179, 19], [90, 133, 167], [94, 165], [97, 72, 93], [99, 31, 136, 128, 185, 2], [102, 191, 137, 20, 145, 162], [106, 180, 105, 164], [107, 11], [119, 192, 181, 184, 91], [123, 18, 176, 115], [124, 127, 131], [132, 14, 5], [135, 186, 140, 101, 155, 80], [138, 81, 62], [139, 126], [141, 182], [149, 111], [150, 45], [151, 74, 69], [152, 29, 96], [173, 7], [175, 169, 134, 168], [178, 33], [189, 113, 109]]

Patient_2 150
#sequences 34 %segments that was sequenced 0.88 longest sequence 6
[[3, 68], [4, 99, 78, 87, 0], [5, 61, 9], [6, 34, 129, 100, 10], [15, 52, 80], [28, 81, 82, 103, 25, 139], [29, 31], [37, 16], [42, 79, 66], [44, 119, 20], [48, 149, 108, 67, 58, 35], [51, 105, 140, 57], [53, 63, 71, 120, 125, 40], [55, 12], [60, 96], [65, 106, 21, 56, 14, 1], [69, 91, 75], [84, 93, 45], [85, 36], [92, 33, 86, 97], [95, 27, 141, 115, 89], [102, 24], [109, 43, 136, 138, 64, 18], [112, 98, 114, 143, 83, 147], [113, 126, 30, 101, 39, 7], [116, 41], [117, 62, 121, 49, 88], [124, 22, 118, 137], [127, 70, 110, 11], [128, 131, 72, 74, 13], [133, 2, 8, 17], [142, 32, 26], [146, 76, 59], [148, 130, 145, 50, 144]]



In [15]:

    
len(set(rows_updated)),len(out_scores),Ntotal









    Out[15]:





(3746, 3935, 3935)



In [9]:

    
out_scores.to_csv(FNAME_OUT, header=True)



In [10]:

    
!paste {FNAME_IN} {FNAME_OUT} | head









    



clip,preictal	clip,preictal
Dog_1_test_segment_0001.mat,0.4538743665290779	Dog_1_test_segment_0001.mat,0.6406154660695936
Dog_1_test_segment_0002.mat,0.1802596928014765	Dog_1_test_segment_0002.mat,0.2970439876069924
Dog_1_test_segment_0003.mat,0.21491236635118124	Dog_1_test_segment_0003.mat,0.3952932347161078
Dog_1_test_segment_0004.mat,0.22850784714844014	Dog_1_test_segment_0004.mat,0.3243250082259081
Dog_1_test_segment_0005.mat,0.2130232242761728	Dog_1_test_segment_0005.mat,0.26647776827366354
Dog_1_test_segment_0006.mat,0.2522360702569839	Dog_1_test_segment_0006.mat,0.3877185115563382
Dog_1_test_segment_0007.mat,0.16566075276892955	Dog_1_test_segment_0007.mat,0.16566075276892955
Dog_1_test_segment_0008.mat,0.28611734296043245	Dog_1_test_segment_0008.mat,0.4483227954427355
Dog_1_test_segment_0009.mat,0.1435247467328573	Dog_1_test_segment_0009.mat,0.17698701740912653



In [11]:

    
out_scores['Dog_2_test_segment_0004.mat']









    Out[11]:





0.3528945711430963



In [12]:

    
df = pd.DataFrame()
df['in'] = pd.read_csv(FNAME_IN, index_col='clip', squeeze=True) #64
df['out'] = pd.read_csv(FNAME_OUT, index_col='clip', squeeze=True)



In [13]:

    
pd.scatter_matrix(df,figsize=(6, 6), diagonal='kde');



In [ ]: