Evaluation of off-the-shelf coreference models

In [1]:

    

%%capture
%load_ext autoreload
%autoreload 2
%matplotlib inline


    

In [2]:

    

import os
import sys
from pathlib import Path

import matplotlib.pyplot as plt


    

In [3]:

    

sys.path.append(r'../')


    

In [4]:

    

from quillnlp.utils import extract_referents_from_xml_tagged_strings


    

In [5]:

    

DATA_DIR = Path("../data")
COREF_TEST_DATA = DATA_DIR / "validated/pronom_coref_res/junkfood_coref_gold.tsv"

texts = []
with open(COREF_TEST_DATA) as i:
    for line in i:
        line = line.strip().split("\t")
        text = " ".join(line)
        texts.append(text)
        
data = list(extract_referents_from_xml_tagged_strings(texts))


    

    




[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 72, 'end': 93, 'str_': 'more nutritious foods', 'id': '2'}, {'start': 102, 'end': 106, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 78, 'end': 83, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 123, 'end': 134, 'str_': 'these goods', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 128, 'end': 135, 'str_': 'schools', 'id': '0'}, {'start': 143, 'end': 147, 'str_': 'them', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 63, 'end': 65, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 65, 'end': 69, 'str_': 'they', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 101, 'end': 123, 'str_': 'advertising in schools', 'id': '2'}, {'start': 127, 'end': 129, 'str_': 'it', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'It', 'id': '1'}, {'start': 111, 'end': 115, 'str_': 'kids', 'id': '2'}, {'start': 121, 'end': 126, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 61, 'str_': 'I', 'id': '2'}, {'start': 139, 'end': 141, 'str_': 'it', 'id': '1'}, {'start': 174, 'end': 178, 'str_': 'them', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'this', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'we', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 75, 'end': 79, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 71, 'end': 75, 'str_': 'they', 'id': '0'}, {'start': 86, 'end': 88, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 100, 'end': 102, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'It', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}, {'start': 73, 'end': 82, 'str_': 'the child', 'id': '2'}, {'start': 100, 'end': 104, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 129, 'end': 131, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'we', 'id': '2'}, {'start': 111, 'end': 115, 'str_': 'them', 'id': '3'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 97, 'end': 102, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 76, 'end': 80, 'str_': 'some', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'kids', 'id': '2'}, {'start': 87, 'end': 91, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 61, 'str_': 'I', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'this', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}, {'start': 103, 'end': 107, 'str_': 'kids', 'id': '2'}, {'start': 118, 'end': 123, 'str_': 'their', 'id': '2'}, {'start': 128, 'end': 137, 'str_': 'junk food', 'id': '1'}, {'start': 141, 'end': 145, 'str_': 'they', 'id': '2'}, {'start': 161, 'end': 163, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'kids', 'id': '2'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 95, 'end': 99, 'str_': 'they', 'id': '2'}, {'start': 114, 'end': 116, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 113, 'end': 118, 'str_': 'these', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 130, 'end': 134, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 80, 'end': 82, 'str_': 'it', 'id': '1'}, {'start': 103, 'end': 105, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 78, 'end': 82, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 68, 'end': 70, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}, {'start': 159, 'end': 164, 'str_': 'their', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 82, 'end': 84, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 82, 'end': 84, 'str_': 'it', 'id': '1'}, {'start': 123, 'end': 127, 'str_': 'kids', 'id': '2'}, {'start': 145, 'end': 149, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 107, 'end': 109, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 83, 'end': 87, 'str_': 'them', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 68, 'end': 72, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 68, 'str_': 'students', 'id': '2'}, {'start': 98, 'end': 102, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'kids', 'id': '2'}, {'start': 118, 'end': 122, 'str_': 'they', 'id': '2'}, {'start': 143, 'end': 148, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 174, 'end': 178, 'str_': 'them', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 68, 'end': 72, 'str_': 'they', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 73, 'end': 77, 'str_': 'this', 'id': '-1'}, {'start': 78, 'end': 82, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 63, 'end': 65, 'str_': 'it', 'id': '1'}, {'start': 91, 'end': 93, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 63, 'end': 65, 'str_': 'it', 'id': '1'}, {'start': 92, 'end': 94, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 96, 'end': 101, 'str_': 'their', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'It', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'we', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 96, 'end': 101, 'str_': 'their', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 71, 'end': 73, 'str_': 'it', 'id': '-1'}, {'start': 98, 'end': 105, 'str_': 'student', 'id': '2'}, {'start': 111, 'end': 115, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'this', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 68, 'str_': 'students', 'id': '2'}, {'start': 101, 'end': 105, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 61, 'str_': 'I', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 65, 'end': 73, 'str_': 'the food', 'id': '1'}, {'start': 104, 'end': 106, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '-1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 61, 'str_': 'I', 'id': '-1'}, {'start': 91, 'end': 99, 'str_': 'students', 'id': '2'}, {'start': 129, 'end': 133, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 100, 'end': 109, 'str_': 'junk food', 'id': '1'}, {'start': 117, 'end': 119, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 78, 'end': 86, 'str_': 'students', 'id': '2'}, {'start': 102, 'end': 106, 'str_': 'they', 'id': '2'}, {'start': 123, 'end': 128, 'str_': 'their', 'id': '2'}, {'start': 143, 'end': 147, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 70, 'end': 75, 'str_': 'their', 'id': '0'}, {'start': 92, 'end': 96, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '0'}, {'start': 115, 'end': 117, 'str_': 'it', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 99, 'end': 101, 'str_': 'it', 'id': '1'}, {'start': 124, 'end': 128, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}, {'start': 99, 'end': 107, 'str_': 'children', 'id': '2'}, {'start': 131, 'end': 136, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 73, 'end': 81, 'str_': 'children', 'id': '2'}, {'start': 91, 'end': 93, 'str_': 'it', 'id': '1'}, {'start': 97, 'end': 102, 'str_': 'their', 'id': '2'}, {'start': 118, 'end': 123, 'str_': 'their', 'id': '2'}, {'start': 137, 'end': 139, 'str_': 'it', 'id': '1'}, {'start': 143, 'end': 148, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '-1'}, {'start': 74, 'end': 82, 'str_': 'children', 'id': '2'}, {'start': 106, 'end': 111, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 124, 'end': 126, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 68, 'str_': 'students', 'id': '2'}, {'start': 119, 'end': 123, 'str_': 'they', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 67, 'str_': 'parents', 'id': '2'}, {'start': 103, 'end': 107, 'str_': 'they', 'id': '2'}, {'start': 116, 'end': 121, 'str_': 'their', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '-1'}, {'start': 114, 'end': 121, 'str_': 'healthy', 'id': '2'}, {'start': 139, 'end': 144, 'str_': 'those', 'id': '2'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 78, 'end': 80, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 151, 'end': 154, 'str_': 'any', 'id': '?'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 68, 'str_': 'they', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 84, 'end': 86, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 70, 'end': 74, 'str_': 'them', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'this', 'id': '-1'}, {'start': 94, 'end': 98, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 64, 'end': 66, 'str_': 'it', 'id': '1'}, {'start': 84, 'end': 89, 'str_': 'their', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 62, 'str_': 'it', 'id': '-1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 0, 'end': 7, 'str_': 'Schools', 'id': '0'}, {'start': 25, 'end': 34, 'str_': 'junk food', 'id': '1'}, {'start': 60, 'end': 64, 'str_': 'they', 'id': '0'}]

In [6]:

    

context = "Students love junk food."
texts_with_context = [context + " " + text for text in texts]
data_with_context = list(extract_referents_from_xml_tagged_strings(texts_with_context))

data_with_context[:3]


    

    




[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 97, 'end': 118, 'str_': 'more nutritious foods', 'id': '2'}, {'start': 127, 'end': 131, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 103, 'end': 108, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 148, 'end': 159, 'str_': 'these goods', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 153, 'end': 160, 'str_': 'schools', 'id': '0'}, {'start': 168, 'end': 172, 'str_': 'them', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 88, 'end': 90, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 90, 'end': 94, 'str_': 'they', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 126, 'end': 148, 'str_': 'advertising in schools', 'id': '2'}, {'start': 152, 'end': 154, 'str_': 'it', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'It', 'id': '1'}, {'start': 136, 'end': 140, 'str_': 'kids', 'id': '2'}, {'start': 146, 'end': 151, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 86, 'str_': 'I', 'id': '2'}, {'start': 164, 'end': 166, 'str_': 'it', 'id': '1'}, {'start': 199, 'end': 203, 'str_': 'them', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'this', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'we', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 100, 'end': 104, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 96, 'end': 100, 'str_': 'they', 'id': '0'}, {'start': 111, 'end': 113, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 125, 'end': 127, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'It', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}, {'start': 98, 'end': 107, 'str_': 'the child', 'id': '2'}, {'start': 125, 'end': 129, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 154, 'end': 156, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'we', 'id': '2'}, {'start': 136, 'end': 140, 'str_': 'them', 'id': '3'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 122, 'end': 127, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 101, 'end': 105, 'str_': 'some', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'kids', 'id': '2'}, {'start': 112, 'end': 116, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 86, 'str_': 'I', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'this', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}, {'start': 128, 'end': 132, 'str_': 'kids', 'id': '2'}, {'start': 143, 'end': 148, 'str_': 'their', 'id': '2'}, {'start': 153, 'end': 162, 'str_': 'junk food', 'id': '1'}, {'start': 166, 'end': 170, 'str_': 'they', 'id': '2'}, {'start': 186, 'end': 188, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'kids', 'id': '2'}, {'start': 114, 'end': 116, 'str_': 'it', 'id': '1'}, {'start': 120, 'end': 124, 'str_': 'they', 'id': '2'}, {'start': 139, 'end': 141, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 138, 'end': 143, 'str_': 'these', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 155, 'end': 159, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 105, 'end': 107, 'str_': 'it', 'id': '1'}, {'start': 128, 'end': 130, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 103, 'end': 107, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 93, 'end': 95, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}, {'start': 184, 'end': 189, 'str_': 'their', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 107, 'end': 109, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 107, 'end': 109, 'str_': 'it', 'id': '1'}, {'start': 148, 'end': 152, 'str_': 'kids', 'id': '2'}, {'start': 170, 'end': 174, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 132, 'end': 134, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 108, 'end': 112, 'str_': 'them', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 93, 'end': 97, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 93, 'str_': 'students', 'id': '2'}, {'start': 123, 'end': 127, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'kids', 'id': '2'}, {'start': 143, 'end': 147, 'str_': 'they', 'id': '2'}, {'start': 168, 'end': 173, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 199, 'end': 203, 'str_': 'them', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 93, 'end': 97, 'str_': 'they', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 98, 'end': 102, 'str_': 'this', 'id': '-1'}, {'start': 103, 'end': 107, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 88, 'end': 90, 'str_': 'it', 'id': '1'}, {'start': 116, 'end': 118, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 88, 'end': 90, 'str_': 'it', 'id': '1'}, {'start': 117, 'end': 119, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 121, 'end': 126, 'str_': 'their', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'It', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'we', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 121, 'end': 126, 'str_': 'their', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 96, 'end': 98, 'str_': 'it', 'id': '-1'}, {'start': 123, 'end': 130, 'str_': 'student', 'id': '2'}, {'start': 136, 'end': 140, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'this', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 93, 'str_': 'students', 'id': '2'}, {'start': 126, 'end': 130, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 86, 'str_': 'I', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 90, 'end': 98, 'str_': 'the food', 'id': '1'}, {'start': 129, 'end': 131, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '-1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 86, 'str_': 'I', 'id': '-1'}, {'start': 116, 'end': 124, 'str_': 'students', 'id': '2'}, {'start': 154, 'end': 158, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 125, 'end': 134, 'str_': 'junk food', 'id': '1'}, {'start': 142, 'end': 144, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 103, 'end': 111, 'str_': 'students', 'id': '2'}, {'start': 127, 'end': 131, 'str_': 'they', 'id': '2'}, {'start': 148, 'end': 153, 'str_': 'their', 'id': '2'}, {'start': 168, 'end': 172, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 95, 'end': 100, 'str_': 'their', 'id': '0'}, {'start': 117, 'end': 121, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '0'}, {'start': 140, 'end': 142, 'str_': 'it', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 124, 'end': 126, 'str_': 'it', 'id': '1'}, {'start': 149, 'end': 153, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}, {'start': 124, 'end': 132, 'str_': 'children', 'id': '2'}, {'start': 156, 'end': 161, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 98, 'end': 106, 'str_': 'children', 'id': '2'}, {'start': 116, 'end': 118, 'str_': 'it', 'id': '1'}, {'start': 122, 'end': 127, 'str_': 'their', 'id': '2'}, {'start': 143, 'end': 148, 'str_': 'their', 'id': '2'}, {'start': 162, 'end': 164, 'str_': 'it', 'id': '1'}, {'start': 168, 'end': 173, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 114, 'end': 118, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '-1'}, {'start': 99, 'end': 107, 'str_': 'children', 'id': '2'}, {'start': 131, 'end': 136, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 149, 'end': 151, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 93, 'str_': 'students', 'id': '2'}, {'start': 144, 'end': 148, 'str_': 'they', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 92, 'str_': 'parents', 'id': '2'}, {'start': 128, 'end': 132, 'str_': 'they', 'id': '2'}, {'start': 141, 'end': 146, 'str_': 'their', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '-1'}, {'start': 139, 'end': 146, 'str_': 'healthy', 'id': '2'}, {'start': 164, 'end': 169, 'str_': 'those', 'id': '2'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 103, 'end': 105, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 176, 'end': 179, 'str_': 'any', 'id': '?'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 93, 'str_': 'they', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 109, 'end': 111, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 95, 'end': 99, 'str_': 'them', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'this', 'id': '-1'}, {'start': 119, 'end': 123, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 89, 'end': 91, 'str_': 'it', 'id': '1'}, {'start': 109, 'end': 114, 'str_': 'their', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 87, 'str_': 'it', 'id': '-1'}, {'start': 110, 'end': 114, 'str_': 'they', 'id': '0'}]
[{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'}, {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'}, {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]






    
Out[6]:





[{'text': 'Students love junk food. Schools should not allow junk food to be sold on campus but should sell more nutritious foods because they have great nutritional value',
  'refs': [{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'},
   {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'},
   {'start': 97, 'end': 118, 'str_': 'more nutritious foods', 'id': '2'},
   {'start': 127, 'end': 131, 'str_': 'they', 'id': '2'}]},
 {'text': 'Students love junk food. Schools should not allow junk food to be sold on campus but it generates money for schools',
  'refs': [{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'},
   {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'},
   {'start': 85, 'end': 87, 'str_': 'it', 'id': '1'}]},
 {'text': 'Students love junk food. Schools should not allow junk food to be sold on campus but they should have greater accessibility to nutrient dense foods',
  'refs': [{'start': 25, 'end': 32, 'str_': 'Schools', 'id': '0'},
   {'start': 50, 'end': 59, 'str_': 'junk food', 'id': '1'},
   {'start': 85, 'end': 89, 'str_': 'they', 'id': '0'}]}]

In [7]:

    

import spacy
import neuralcoref

nlp = spacy.load("en")
neuralcoref.add_to_pipe(nlp)


    

    
Out[7]:





<spacy.lang.en.English at 0x125e63c50>

In [8]:

    

from allennlp.predictors.predictor import Predictor

predictor = Predictor.from_path("https://s3-us-west-2.amazonaws.com/allennlp/models/coref-model-2018.02.05.tar.gz")


    

    




/Users/yvespeirsman/anaconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
  "num_layers={}".format(dropout, num_layers))

In [9]:

    

from collections import defaultdict
from copy import deepcopy

def evaluate_links(predicted_links, gold_links):
    """ 
    Evaluates a set of predicted co-reference links against 
    a set of gold co-reference links. 
    """
    tp, fp, fn = 0, 0, 0
    for link in predicted_links:
        if link in gold_links:
            tp += 1
        else:
            fp += 1
    
    for link in gold_links:
        if link not in predicted_links:
            fn += 1
            
    return (tp, fp, fn)


def transform_coref_clusters_to_links(data_item):
    """
    Transforms a data item and transforms its coreference clusters
    to a list of coreference links.  
    """
    gold_clusters = defaultdict(list)
    for ref in data_item["refs"]:
        gold_clusters[ref["id"]].append((ref["start"], ref["str_"]))

    gold_links = []
    for c_id, c_words in gold_clusters.items():
        if len(c_words) > 1:     
            c_words.sort()
            antecedent = c_words[0]
            for other_word in c_words[1:]:
                gold_links.append((antecedent[0], other_word[0]))
    return gold_links


def get_coref_links_from_spacy_doc(doc):
    """
    Takes a spaCy doc on which co-reference has been performed with 
    Huggingface's Neuralcoref system, and gets the coreference links
    from this document.
    """
    predicted_links = []
    print("NeuralCoref output:", doc._.coref_clusters)
    for c in doc._.coref_clusters:
        antecedent = c[0]
        for other_word in c[1:]:
            predicted_links.append((antecedent.start_char, other_word.start_char))
    return predicted_links
            

def get_coref_links_from_allennlp(text, prediction):
    """
    Takes a text and the corresponding output of the AllenNLP 
    coreference model, and gets the coreference links from this input.
    Tokenization is performed with spaCy. This is AllenNLP's standard
    solution and is also Neuralcoref's tokenization method.
    """
    predicted_links = []
    spacy_doc = nlp(text)
    print("AllenNLP Output:", prediction["clusters"])
    for c in prediction["clusters"]:
        antecedent_token_idx = c[0][0]
        for other_word in c[1:]:
            antecedent_char_idx = spacy_doc[antecedent_token_idx].idx
            other_word_token_idx = other_word[0]
            other_word_char_idx = spacy_doc[other_word_token_idx].idx
            predicted_links.append((antecedent_char_idx, other_word_char_idx))
    
    return predicted_links
    

def evaluate_coref(data, model):
    """
    Perform a full evaluation round on our test data.
    Model is either "neuralcoref" or "allennlp".
    """
    tp, fp, fn = 0, 0, 0
    for data_item in data:
        
        if not data_item["text"].endswith("."):
            data_item["text"] += "."
        
        gold_links = transform_coref_clusters_to_links(data_item)
        
        if model == "neuralcoref":
            doc = nlp(data_item["text"])
            predicted_links = get_coref_links_from_spacy_doc(doc)    
        
        elif model == "allennlp":
            prediction = predictor.predict(document=data_item["text"])
            predicted_links = get_coref_links_from_allennlp(data_item["text"], prediction)
            
        else:
            return ValueError(f"Unknown model: {model}")
                    
        print("Predicted links:", predicted_links)
        print("Gold links", gold_links)
        print("\n")

        tp_, fp_, fn_ = evaluate_links(predicted_links, gold_links)

        tp += tp_
        fp += fp_
        fn += fn_

    p = tp/(tp+fp)
    r = tp/(tp+fn)
    f = 2*p*r/(p+r) if p+r > 0 else 0

    print("Precision:", p)
    print("Recall:", r)
    print("F1 score:", f)
    return p, r, f


    

In [10]:

    

p_neuralcoref, r_neuralcoref, f_neuralcoref = evaluate_coref(data, model="neuralcoref")


    

    




NeuralCoref output: [more nutritious foods: [more nutritious foods, they]]
Predicted links: [(72, 102)]
Gold links [(72, 102)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(0, 78)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 123)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [physical education programs in schools: [physical education programs in schools, them]]
Predicted links: [(97, 143)]
Gold links [(0, 128), (0, 143), (25, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 63)]
Gold links [(25, 63)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 65)]
Gold links [(25, 65)]


NeuralCoref output: [Schools: [Schools, they], candy companies: [candy companies, it]]
Predicted links: [(0, 60), (80, 127)]
Gold links [(0, 60), (101, 127)]


NeuralCoref output: [kids: [kids, their]]
Predicted links: [(111, 121)]
Gold links [(25, 60), (111, 121)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [schools: [schools, them]]
Predicted links: [(80, 174)]
Gold links [(0, 174), (25, 139)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 75)]
Gold links [(0, 75)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 71)]
Gold links [(0, 71), (25, 86)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, it, it]]
Predicted links: [(0, 64), (0, 100)]
Gold links [(25, 64), (25, 100)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 100)]
Gold links [(25, 60), (73, 100)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(96, 129)]
Gold links [(25, 129)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [we: [we, our], our kids: [our kids, them]]
Predicted links: [(60, 78), (78, 111)]
Gold links []


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, their], junk food: [junk food, it]]
Predicted links: [(0, 97), (25, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60), (25, 76)]


NeuralCoref output: [kids: [kids, they]]
Predicted links: [(60, 87)]
Gold links [(60, 87)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they, their, they]]
Predicted links: [(0, 64), (0, 118), (0, 141)]
Gold links [(0, 64), (25, 128), (25, 161), (103, 118), (103, 141)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [kids: [kids, they], it: [it, it]]
Predicted links: [(60, 95), (89, 114)]
Gold links [(25, 89), (25, 114), (60, 95)]


NeuralCoref output: [Schools: [Schools, they], junk food: [junk food, these food]]
Predicted links: [(0, 60), (25, 113)]
Gold links [(0, 60), (25, 113)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 130)]
Gold links [(0, 130)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [it: [it, it]]
Predicted links: [(80, 103)]
Gold links [(25, 80), (25, 103)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 78)]
Gold links [(0, 78)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 68)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


NeuralCoref output: [Schools: [Schools, they, their]]
Predicted links: [(0, 64), (0, 159)]
Gold links [(0, 64), (0, 159)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60), (25, 82)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [many people: [many people, they]]
Predicted links: [(60, 145)]
Gold links [(25, 82), (123, 145)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60), (25, 107)]


NeuralCoref output: [Schools: [Schools, them]]
Predicted links: [(0, 83)]
Gold links [(25, 83)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 68)]
Gold links [(0, 68)]


NeuralCoref output: [students: [students, they]]
Predicted links: [(60, 98)]
Gold links [(60, 98)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [kids: [kids, they, their]]
Predicted links: [(64, 118), (64, 143)]
Gold links [(64, 118), (64, 143)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they, them]]
Predicted links: [(0, 60), (0, 174)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 68)]
Gold links []


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 78)]
Gold links [(0, 78)]


NeuralCoref output: [Schools: [Schools, it, it]]
Predicted links: [(0, 63), (0, 91)]
Gold links [(25, 63), (25, 91)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 63), (25, 92)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(0, 96)]
Gold links [(0, 96), (25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(0, 96)]
Gold links [(0, 96), (25, 64)]


NeuralCoref output: [the individual student: [the individual student, they]]
Predicted links: [(83, 111)]
Gold links [(98, 111)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: [students: [students, they]]
Predicted links: [(60, 101)]
Gold links [(60, 101)]


NeuralCoref output: [Schools: [Schools, the schools]]
Predicted links: [(0, 74)]
Gold links []


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [junk food: [junk food, the food, it]]
Predicted links: [(25, 65), (25, 104)]
Gold links [(25, 65), (25, 104)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [students: [students, they]]
Predicted links: [(91, 129)]
Gold links [(91, 129)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 100), (25, 117)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they, they, their, they]]
Predicted links: [(0, 60), (0, 102), (0, 123), (0, 143)]
Gold links [(0, 60), (78, 102), (78, 123), (78, 143)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, its, their, they]]
Predicted links: [(0, 60), (0, 70), (0, 92)]
Gold links [(0, 70), (0, 92)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, it, it]]
Predicted links: [(0, 60), (0, 115)]
Gold links [(0, 60), (0, 115)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they, they]]
Predicted links: [(0, 60), (0, 124)]
Gold links [(0, 60), (0, 124), (25, 99)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they, their]]
Predicted links: [(0, 60), (0, 131)]
Gold links [(0, 60), (99, 131)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [junk food: [junk food, it, it], children: [children, their, their, their]]
Predicted links: [(25, 91), (25, 137), (73, 97), (73, 118), (73, 143)]
Gold links [(25, 91), (25, 137), (73, 97), (73, 118), (73, 143)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 89)]
Gold links [(0, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [children: [children, their]]
Predicted links: [(74, 106)]
Gold links [(74, 106)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [serious consideration: [serious consideration, it]]
Predicted links: [(60, 124)]
Gold links [(25, 124)]


NeuralCoref output: [students: [students, they]]
Predicted links: [(60, 119)]
Gold links [(60, 119)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [parents: [parents, they, their]]
Predicted links: [(60, 103), (60, 116)]
Gold links [(60, 103), (60, 116)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(114, 139)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(25, 78)]
Gold links [(25, 78)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [it: [it, it]]
Predicted links: [(64, 84)]
Gold links [(25, 64), (25, 84)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 60)]
Gold links [(25, 60)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, them]]
Predicted links: [(0, 70)]
Gold links [(25, 70)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


NeuralCoref output: [Schools: [Schools, they], junk food: [junk food, the junk food]]
Predicted links: [(0, 60), (25, 77)]
Gold links [(0, 60)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 94)]
Gold links [(0, 94)]


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(0, 84)]
Gold links [(0, 84), (25, 64)]


NeuralCoref output: []
Predicted links: []
Gold links [(0, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


Precision: 0.6477987421383647
Recall: 0.412
F1 score: 0.5036674816625916

In [11]:

    

p_neuralcoref_context, r_neuralcoref_context, f_neuralcoref_context = evaluate_coref(data_with_context, 
                                                                                     model="neuralcoref")


    

    




NeuralCoref output: [more nutritious foods: [more nutritious foods, they]]
Predicted links: [(97, 127)]
Gold links [(97, 127)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(25, 103)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 148)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [physical education programs in schools: [physical education programs in schools, them]]
Predicted links: [(122, 168)]
Gold links [(25, 153), (25, 168), (50, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 88)]
Gold links [(50, 88)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 90)]
Gold links [(50, 90)]


NeuralCoref output: [Schools: [Schools, they], candy companies: [candy companies, it]]
Predicted links: [(25, 85), (105, 152)]
Gold links [(25, 85), (126, 152)]


NeuralCoref output: [kids: [kids, their]]
Predicted links: [(136, 146)]
Gold links [(50, 85), (136, 146)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [schools: [schools, them]]
Predicted links: [(105, 199)]
Gold links [(25, 199), (50, 164)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 100)]
Gold links [(25, 100)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 96)]
Gold links [(25, 96), (50, 111)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, it, it]]
Predicted links: [(25, 89), (25, 125)]
Gold links [(50, 89), (50, 125)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 125)]
Gold links [(50, 85), (98, 125)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(121, 154)]
Gold links [(50, 154)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [we: [we, our], our kids: [our kids, them]]
Predicted links: [(85, 103), (103, 136)]
Gold links []


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, their], junk food: [junk food, it]]
Predicted links: [(25, 122), (50, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85), (50, 101)]


NeuralCoref output: [kids: [kids, they]]
Predicted links: [(85, 112)]
Gold links [(85, 112)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they, their, they]]
Predicted links: [(25, 89), (25, 143), (25, 166)]
Gold links [(25, 89), (50, 153), (50, 186), (128, 143), (128, 166)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [kids: [kids, they], it: [it, it]]
Predicted links: [(85, 120), (114, 139)]
Gold links [(50, 114), (50, 139), (85, 120)]


NeuralCoref output: [Schools: [Schools, they], junk food: [junk food, these food]]
Predicted links: [(25, 85), (50, 138)]
Gold links [(25, 85), (50, 138)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 155)]
Gold links [(25, 155)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [it: [it, it]]
Predicted links: [(105, 128)]
Gold links [(50, 105), (50, 128)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 103)]
Gold links [(25, 103)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 93)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


NeuralCoref output: [Schools: [Schools, they, their]]
Predicted links: [(25, 89), (25, 184)]
Gold links [(25, 89), (25, 184)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85), (50, 107)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [many people: [many people, they]]
Predicted links: [(85, 170)]
Gold links [(50, 107), (148, 170)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85), (50, 132)]


NeuralCoref output: [Schools: [Schools, them]]
Predicted links: [(25, 108)]
Gold links [(50, 108)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 93)]
Gold links [(25, 93)]


NeuralCoref output: [students: [students, they]]
Predicted links: [(85, 123)]
Gold links [(85, 123)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [kids: [kids, they, their]]
Predicted links: [(89, 143), (89, 168)]
Gold links [(89, 143), (89, 168)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they, them]]
Predicted links: [(25, 85), (25, 199)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 93)]
Gold links []


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 103)]
Gold links [(25, 103)]


NeuralCoref output: [Schools: [Schools, it, it]]
Predicted links: [(25, 88), (25, 116)]
Gold links [(50, 88), (50, 116)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 88), (50, 117)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(25, 121)]
Gold links [(25, 121), (50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(25, 121)]
Gold links [(25, 121), (50, 89)]


NeuralCoref output: [the individual student: [the individual student, they]]
Predicted links: [(108, 136)]
Gold links [(123, 136)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: [students: [students, they]]
Predicted links: [(85, 126)]
Gold links [(85, 126)]


NeuralCoref output: [Schools: [Schools, the schools]]
Predicted links: [(25, 99)]
Gold links []


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [junk food: [junk food, the food, it]]
Predicted links: [(50, 90), (50, 129)]
Gold links [(50, 90), (50, 129)]


NeuralCoref output: []
Predicted links: []
Gold links []


NeuralCoref output: [students: [students, they]]
Predicted links: [(116, 154)]
Gold links [(116, 154)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 125), (50, 142)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they, they, their, they]]
Predicted links: [(25, 85), (25, 127), (25, 148), (25, 168)]
Gold links [(25, 85), (103, 127), (103, 148), (103, 168)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, its, their, they]]
Predicted links: [(25, 85), (25, 95), (25, 117)]
Gold links [(25, 95), (25, 117)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, it, it]]
Predicted links: [(25, 85), (25, 140)]
Gold links [(25, 85), (25, 140)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they, they]]
Predicted links: [(25, 85), (25, 149)]
Gold links [(25, 85), (25, 149), (50, 124)]


NeuralCoref output: [Students: [Students, the students], Schools: [Schools, they]]
Predicted links: [(0, 104), (25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they, their]]
Predicted links: [(25, 85), (25, 156)]
Gold links [(25, 85), (124, 156)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [junk food: [junk food, it, it], children: [children, their, their, their]]
Predicted links: [(50, 116), (50, 162), (98, 122), (98, 143), (98, 168)]
Gold links [(50, 116), (50, 162), (98, 122), (98, 143), (98, 168)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 114)]
Gold links [(25, 114)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [children: [children, their]]
Predicted links: [(99, 131)]
Gold links [(99, 131)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [serious consideration: [serious consideration, it]]
Predicted links: [(85, 149)]
Gold links [(50, 149)]


NeuralCoref output: [students: [students, they]]
Predicted links: [(85, 144)]
Gold links [(85, 144)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [parents: [parents, they, their]]
Predicted links: [(85, 128), (85, 141)]
Gold links [(85, 128), (85, 141)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(139, 164)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


NeuralCoref output: [junk food: [junk food, it]]
Predicted links: [(50, 103)]
Gold links [(50, 103)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [it: [it, it]]
Predicted links: [(89, 109)]
Gold links [(50, 89), (50, 109)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 85)]
Gold links [(50, 85)]


NeuralCoref output: []
Predicted links: []
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, them]]
Predicted links: [(25, 95)]
Gold links [(50, 95)]


NeuralCoref output: [Schools: [Schools, it]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


NeuralCoref output: [Schools: [Schools, they], junk food: [junk food, the junk food]]
Predicted links: [(25, 85), (50, 102)]
Gold links [(25, 85)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 119)]
Gold links [(25, 119)]


NeuralCoref output: [Schools: [Schools, their]]
Predicted links: [(25, 109)]
Gold links [(25, 109), (50, 89)]


NeuralCoref output: []
Predicted links: []
Gold links [(25, 110)]


NeuralCoref output: [Schools: [Schools, they]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


Precision: 0.64375
Recall: 0.412
F1 score: 0.5024390243902439

In [12]:

    

p_allennlp, r_allennlp, f_allennlp = evaluate_coref(data, model="allennlp")


    

    




Encountered the antecedent_indices key in the model's return dictionary which couldn't be split by the batch size. Key will be ignored.






    




AllenNLP Output: [[[14, 16], [18, 18]]]
Predicted links: [(72, 102)]
Gold links [(72, 102)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 123)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]], [[23, 23], [26, 26]]]
Predicted links: [(25, 64), (128, 143)]
Gold links [(0, 128), (0, 143), (25, 64)]


AllenNLP Output: [[[4, 5], [13, 13]]]
Predicted links: [(25, 63)]
Gold links [(25, 63)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 65)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60), (101, 127)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 60), (111, 121)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [26, 26]], [[17, 17], [32, 32]]]
Predicted links: [(25, 139), (80, 174)]
Gold links [(0, 174), (25, 139)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: []
Predicted links: []
Gold links [(0, 75)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(30, 86)]
Gold links [(0, 71), (25, 86)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12], [18, 18]]]
Predicted links: [(25, 64), (25, 100)]
Gold links [(25, 64), (25, 100)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[15, 17], [21, 21]]]
Predicted links: [(73, 100)]
Gold links [(25, 60), (73, 100)]


AllenNLP Output: [[[18, 19], [24, 24]]]
Predicted links: [(96, 129)]
Gold links [(25, 129)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[12, 12], [16, 16]], [[16, 17], [22, 22]]]
Predicted links: [(60, 78), (78, 111)]
Gold links []


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60), (25, 76)]


AllenNLP Output: [[[12, 12], [17, 17]]]
Predicted links: [(60, 87)]
Gold links [(60, 87)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[3, 3], [12, 12]]]
Predicted links: [(19, 64)]
Gold links []


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[12, 12], [21, 21], [26, 26]], [[4, 5], [31, 31]]]
Predicted links: [(64, 118), (64, 141), (25, 161)]
Gold links [(0, 64), (25, 128), (25, 161), (103, 118), (103, 141)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [18, 18], [24, 24]], [[12, 12], [20, 20]]]
Predicted links: [(25, 89), (25, 114), (60, 95)]
Gold links [(25, 89), (25, 114), (60, 95)]


AllenNLP Output: [[[5, 5], [21, 22]]]
Predicted links: [(30, 113)]
Gold links [(0, 60), (25, 113)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 130)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [16, 16], [20, 20]]]
Predicted links: [(25, 80), (25, 103)]
Gold links [(25, 80), (25, 103)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[3, 3], [12, 12]]]
Predicted links: [(19, 60)]
Gold links [(0, 78)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [14, 14]]]
Predicted links: [(25, 68)]
Gold links [(25, 68)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: [[[12, 12], [29, 29]]]
Predicted links: [(64, 159)]
Gold links [(0, 64), (0, 159)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [16, 16]]]
Predicted links: [(25, 82)]
Gold links [(0, 60), (25, 82)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [16, 16]], [[24, 24], [29, 29]]]
Predicted links: [(25, 82), (123, 145)]
Gold links [(25, 82), (123, 145)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]], [[4, 5], [21, 21]]]
Predicted links: [(0, 60), (25, 107)]
Gold links [(0, 60), (25, 107)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 83)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 68)]


AllenNLP Output: []
Predicted links: []
Gold links [(60, 98)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links []


AllenNLP Output: [[[12, 12], [22, 22], [27, 27]]]
Predicted links: [(64, 118), (64, 143)]
Gold links [(64, 118), (64, 143)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]], [[15, 22], [36, 36]]]
Predicted links: [(0, 60), (77, 174)]
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: []
Predicted links: []
Gold links [(0, 78)]


AllenNLP Output: [[[4, 5], [13, 13], [20, 20]]]
Predicted links: [(25, 63), (25, 91)]
Gold links [(25, 63), (25, 91)]


AllenNLP Output: [[[4, 5], [13, 13], [19, 19]]]
Predicted links: [(25, 63), (25, 92)]
Gold links [(25, 63), (25, 92)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(0, 96), (25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[4, 5], [12, 12]], [[0, 0], [20, 20]]]
Predicted links: [(25, 64), (0, 96)]
Gold links [(0, 96), (25, 64)]


AllenNLP Output: [[[17, 19], [21, 21]]]
Predicted links: [(83, 111)]
Gold links [(98, 111)]


AllenNLP Output: [[[3, 3], [12, 12]]]
Predicted links: [(19, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: [[[12, 12], [19, 19]]]
Predicted links: [(60, 101)]
Gold links [(60, 101)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [21, 21]]]
Predicted links: [(25, 104)]
Gold links [(25, 65), (25, 104)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[17, 17], [24, 24]]]
Predicted links: [(91, 129)]
Gold links [(91, 129)]


AllenNLP Output: [[[18, 19], [22, 22]]]
Predicted links: [(100, 117)]
Gold links [(25, 100), (25, 117)]


AllenNLP Output: [[[0, 0], [12, 12]], [[4, 5], [24, 26]]]
Predicted links: [(0, 60), (25, 130)]
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[12, 12], [19, 19], [23, 23], [27, 27]]]
Predicted links: [(60, 102), (60, 123), (60, 143)]
Gold links [(0, 60), (78, 102), (78, 123), (78, 143)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12], [14, 14], [18, 18]]]
Predicted links: [(0, 60), (0, 70), (0, 92)]
Gold links [(0, 70), (0, 92)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12], [22, 22]]]
Predicted links: [(25, 60), (25, 115)]
Gold links [(0, 60), (0, 115)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [20, 20]], [[12, 12], [25, 25]]]
Predicted links: [(25, 99), (60, 124)]
Gold links [(0, 60), (0, 124), (25, 99)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]], [[15, 17], [28, 29]]]
Predicted links: [(0, 60), (78, 134)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]], [[18, 26], [24, 24]]]
Predicted links: [(0, 60), (99, 131)]
Gold links [(0, 60), (99, 131)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [17, 17], [26, 26]], [[14, 14], [19, 19], [23, 23], [28, 28]]]
Predicted links: [(25, 91), (25, 137), (73, 97), (73, 118), (73, 143)]
Gold links [(25, 91), (25, 137), (73, 97), (73, 118), (73, 143)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 89)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]], [[14, 14], [19, 19]]]
Predicted links: [(25, 64), (74, 106)]
Gold links [(74, 106)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [21, 21]]]
Predicted links: [(25, 124)]
Gold links [(25, 124)]


AllenNLP Output: []
Predicted links: []
Gold links [(60, 119)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 60)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: [[[12, 12], [19, 19], [22, 22]]]
Predicted links: [(60, 103), (60, 116)]
Gold links [(60, 103), (60, 116)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(114, 139)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 64)]
Gold links [(0, 64)]


AllenNLP Output: [[[4, 5], [15, 15]]]
Predicted links: [(25, 78)]
Gold links [(25, 78)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 60)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[4, 5], [12, 12], [16, 16]]]
Predicted links: [(25, 64), (25, 84)]
Gold links [(25, 64), (25, 84)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 60)]
Gold links [(25, 60)]


AllenNLP Output: [[[5, 5], [12, 12]]]
Predicted links: [(30, 64)]
Gold links [(25, 64)]


AllenNLP Output: []
Predicted links: []
Gold links [(25, 70)]


AllenNLP Output: [[[4, 5], [12, 12]]]
Predicted links: [(25, 64)]
Gold links [(25, 64)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 94)]


AllenNLP Output: []
Predicted links: []
Gold links [(0, 84), (25, 64)]


AllenNLP Output: [[[0, 0], [16, 16]]]
Predicted links: [(0, 85)]
Gold links [(0, 85)]


AllenNLP Output: [[[0, 0], [12, 12]]]
Predicted links: [(0, 60)]
Gold links [(0, 60)]


Precision: 0.7676767676767676
Recall: 0.608
F1 score: 0.6785714285714285

In [13]:

    

p_allennlp_context, r_allennlp_context, f_allennlp_context = evaluate_coref(data_with_context, model="allennlp")


    

    




AllenNLP Output: [[[19, 21], [23, 23]]]
Predicted links: [(97, 127)]
Gold links [(97, 127)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]], [[5, 5], [21, 21]]]
Predicted links: [(50, 89), (25, 103)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: []
Predicted links: []
Gold links [(50, 148)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]], [[28, 28], [31, 31]]]
Predicted links: [(50, 89), (153, 168)]
Gold links [(25, 153), (25, 168), (50, 89)]


AllenNLP Output: [[[9, 10], [18, 18]]]
Predicted links: [(50, 88)]
Gold links [(50, 88)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [18, 18]]]
Predicted links: [(25, 90)]
Gold links [(50, 90)]


AllenNLP Output: [[[5, 5], [17, 17]], [[9, 10], [28, 28]]]
Predicted links: [(25, 85), (50, 152)]
Gold links [(25, 85), (126, 152)]


AllenNLP Output: [[[9, 10], [17, 17]], [[5, 5], [28, 28]]]
Predicted links: [(50, 85), (25, 146)]
Gold links [(50, 85), (136, 146)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [31, 31]]]
Predicted links: [(50, 164)]
Gold links [(25, 199), (50, 164)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: []
Predicted links: []
Gold links [(50, 89)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[5, 5], [20, 20]]]
Predicted links: [(25, 100)]
Gold links [(25, 100)]


AllenNLP Output: [[[5, 5], [19, 19]], [[10, 10], [22, 22]]]
Predicted links: [(25, 96), (55, 111)]
Gold links [(25, 96), (50, 111)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17], [23, 23]]]
Predicted links: [(50, 89), (50, 125)]
Gold links [(50, 89), (50, 125)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [26, 26]]]
Predicted links: [(25, 125)]
Gold links [(50, 85), (98, 125)]


AllenNLP Output: [[[9, 10], [29, 29]]]
Predicted links: [(50, 154)]
Gold links [(50, 154)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[17, 17], [21, 21]], [[21, 22], [27, 27]]]
Predicted links: [(85, 103), (103, 136)]
Gold links []


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]], [[5, 5], [24, 24]]]
Predicted links: [(50, 89), (25, 122)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85), (50, 101)]


AllenNLP Output: [[[17, 17], [22, 22]]]
Predicted links: [(85, 112)]
Gold links [(85, 112)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17], [26, 26], [31, 31]], [[9, 10], [36, 36]]]
Predicted links: [(25, 89), (25, 143), (25, 166), (50, 186)]
Gold links [(25, 89), (50, 153), (50, 186), (128, 143), (128, 166)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [23, 23], [29, 29]], [[5, 5], [25, 25]]]
Predicted links: [(50, 114), (50, 139), (25, 120)]
Gold links [(50, 114), (50, 139), (85, 120)]


AllenNLP Output: [[[5, 5], [17, 17]], [[9, 10], [26, 27]]]
Predicted links: [(25, 85), (50, 138)]
Gold links [(25, 85), (50, 138)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[5, 5], [29, 29]]]
Predicted links: [(25, 155)]
Gold links [(25, 155)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [21, 21], [25, 25]]]
Predicted links: [(50, 105), (50, 128)]
Gold links [(50, 105), (50, 128)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[13, 13], [17, 17]], [[5, 5], [21, 21]]]
Predicted links: [(66, 85), (25, 103)]
Gold links [(25, 103)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [19, 19]]]
Predicted links: [(50, 93)]
Gold links [(50, 93)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[5, 5], [17, 17], [34, 34]]]
Predicted links: [(25, 89), (25, 184)]
Gold links [(25, 89), (25, 184)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]], [[9, 10], [21, 21]]]
Predicted links: [(25, 85), (50, 107)]
Gold links [(25, 85), (50, 107)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [21, 21]], [[29, 29], [34, 34]]]
Predicted links: [(50, 107), (148, 170)]
Gold links [(50, 107), (148, 170)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: []
Predicted links: []
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]], [[9, 10], [26, 26]]]
Predicted links: [(25, 85), (50, 132)]
Gold links [(25, 85), (50, 132)]


AllenNLP Output: [[[5, 5], [20, 20]]]
Predicted links: [(25, 108)]
Gold links [(50, 108)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [19, 19]]]
Predicted links: [(25, 93)]
Gold links [(25, 93)]


AllenNLP Output: [[[5, 5], [24, 24]]]
Predicted links: [(25, 123)]
Gold links [(85, 123)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links []


AllenNLP Output: [[[17, 17], [27, 27], [32, 32]]]
Predicted links: [(89, 143), (89, 168)]
Gold links [(89, 143), (89, 168)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]], [[20, 27], [41, 41]]]
Predicted links: [(25, 85), (102, 199)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [19, 19]]]
Predicted links: [(25, 93)]
Gold links []


AllenNLP Output: [[[5, 5], [20, 20]]]
Predicted links: [(25, 103)]
Gold links [(25, 103)]


AllenNLP Output: [[[9, 10], [18, 18], [25, 25]]]
Predicted links: [(50, 88), (50, 116)]
Gold links [(50, 88), (50, 116)]


AllenNLP Output: [[[9, 10], [18, 18], [24, 24]]]
Predicted links: [(50, 88), (50, 117)]
Gold links [(50, 88), (50, 117)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]], [[5, 5], [22, 22]]]
Predicted links: [(50, 89), (25, 121)]
Gold links [(25, 121), (50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[9, 10], [17, 17]], [[5, 5], [25, 25]]]
Predicted links: [(50, 89), (25, 121)]
Gold links [(25, 121), (50, 89)]


AllenNLP Output: [[[22, 24], [26, 26]]]
Predicted links: [(108, 136)]
Gold links [(123, 136)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[5, 5], [24, 24]]]
Predicted links: [(25, 126)]
Gold links [(85, 126)]


AllenNLP Output: []
Predicted links: []
Gold links []


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [26, 26]]]
Predicted links: [(50, 129)]
Gold links [(50, 90), (50, 129)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links []


AllenNLP Output: [[[22, 22], [29, 29]]]
Predicted links: [(116, 154)]
Gold links [(116, 154)]


AllenNLP Output: [[[9, 10], [27, 27]]]
Predicted links: [(50, 142)]
Gold links [(50, 125), (50, 142)]


AllenNLP Output: [[[5, 5], [17, 17]], [[9, 10], [29, 31]]]
Predicted links: [(25, 85), (50, 155)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17], [24, 24], [28, 28], [32, 32]]]
Predicted links: [(25, 85), (25, 127), (25, 148), (25, 168)]
Gold links [(25, 85), (103, 127), (103, 148), (103, 168)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17], [19, 19], [23, 23]]]
Predicted links: [(25, 85), (25, 95), (25, 117)]
Gold links [(25, 95), (25, 117)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17], [27, 27]]]
Predicted links: [(50, 85), (50, 140)]
Gold links [(25, 85), (25, 140)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17], [30, 30]], [[9, 10], [25, 25]]]
Predicted links: [(25, 85), (25, 149), (50, 124)]
Gold links [(25, 85), (25, 149), (50, 124)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]], [[20, 22], [33, 34]]]
Predicted links: [(25, 85), (103, 159)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17], [29, 29]]]
Predicted links: [(25, 85), (25, 156)]
Gold links [(25, 85), (124, 156)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [22, 22], [31, 31]], [[5, 5], [24, 24], [28, 28], [33, 33]]]
Predicted links: [(50, 116), (50, 162), (25, 122), (25, 143), (25, 168)]
Gold links [(50, 116), (50, 162), (98, 122), (98, 143), (98, 168)]


AllenNLP Output: [[[5, 5], [21, 21]]]
Predicted links: [(25, 114)]
Gold links [(25, 114)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]], [[19, 19], [24, 24]]]
Predicted links: [(50, 89), (99, 131)]
Gold links [(99, 131)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [26, 26]]]
Predicted links: [(50, 149)]
Gold links [(50, 149)]


AllenNLP Output: [[[5, 5], [28, 28]]]
Predicted links: [(25, 144)]
Gold links [(85, 144)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[17, 17], [24, 24], [27, 27]]]
Predicted links: [(85, 128), (85, 141)]
Gold links [(85, 128), (85, 141)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(139, 164)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(25, 89)]


AllenNLP Output: [[[9, 10], [20, 20]]]
Predicted links: [(50, 103)]
Gold links [(50, 103)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[9, 10], [17, 17], [21, 21]]]
Predicted links: [(50, 89), (50, 109)]
Gold links [(50, 89), (50, 109)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 85)]
Gold links [(50, 85)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [19, 19]]]
Predicted links: [(25, 95)]
Gold links [(50, 95)]


AllenNLP Output: [[[9, 10], [17, 17]]]
Predicted links: [(50, 89)]
Gold links [(50, 89)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


AllenNLP Output: [[[5, 5], [22, 22]]]
Predicted links: [(25, 119)]
Gold links [(25, 119)]


AllenNLP Output: [[[9, 10], [17, 17]], [[5, 5], [22, 22]]]
Predicted links: [(50, 89), (25, 109)]
Gold links [(25, 109), (50, 89)]


AllenNLP Output: [[[5, 5], [21, 21]]]
Predicted links: [(25, 110)]
Gold links [(25, 110)]


AllenNLP Output: [[[5, 5], [17, 17]]]
Predicted links: [(25, 85)]
Gold links [(25, 85)]


Precision: 0.8300395256916996
Recall: 0.84
F1 score: 0.8349900596421471

In [14]:

    

import numpy as np
import matplotlib.pyplot as plt
 
plt.rcParams['figure.figsize'] = [12, 8]    

barWidth = 0.15
 
bars1 = [p_neuralcoref, r_neuralcoref, f_neuralcoref]
bars2 = [p_neuralcoref_context, r_neuralcoref_context, f_neuralcoref_context]
bars3 = [p_allennlp, r_allennlp, f_allennlp]
bars4 = [p_allennlp_context, r_allennlp_context, f_allennlp_context]
 
# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
r3 = [x + 2*barWidth for x in r1]
r4 = [x + 3*barWidth for x in r1]
 
# Make the plot
plt.bar(r1, bars1, color='lightblue', width=barWidth, edgecolor='white', label='Neuralcoref')
plt.bar(r2, bars2, color='darkblue', width=barWidth, edgecolor='white', label='Neuralcoref + context')
plt.bar(r3, bars3, color='lightgreen', width=barWidth, edgecolor='white', label='AllenNLP')
plt.bar(r4, bars4, color='darkgreen', width=barWidth, edgecolor='white', label='AllenNLP + context')
 
# Add xticks on the middle of the group bars
plt.xlabel('Co-reference Performance', fontweight='bold')
plt.xticks([r + 1.5 * barWidth for r in range(len(bars1))], ['Precision', 'Recall', 'F-score'])
 
# Create legend & Show graphic
plt.legend(loc='lower right')
plt.show()