In [1]:
import os
import processors
from processors import *
from processors.visualization import JupyterVisualizer as viz
print(processors.__version__)
In [2]:
API = ProcessorsAPI(port=8886, keep_alive=True)
In [3]:
demo_dir = os.path.split(os.getcwd())[0]
docs_dir = os.path.join(demo_dir, "docs")
grammar_dir = os.path.join(demo_dir, "grammar")
print("DEMO DIRECTORY:\t{}".format(demo_dir))
print("DOCS DIRECTORY:\t{}".format(docs_dir))
print("GRAMMAR DIRECTORY:\t{}".format(grammar_dir))
In [4]:
# TODO: Add to py-processors
def display_mentions(mentions):
for m in mentions:
print(m.document.id if m.document.id else id(m.document))
viz.display_mention(m)
def deserialize_document(doc):
if json_file.endswith(".json"):
fpath = os.path.join(serialized_docs_dir, json_file)
with open(fpath) as infile:
text = infile.read()
doc_dict = json.loads(text)
doc = Document.load_from_JSON(doc_dict)
# FIXED, but not released (ADD TEST)
# py-processors v3.0.3 wasn't retrieving the doc id
doc.id = os.path.splitext(json_file)[0]
return doc
else:
return None
def deserialize_documents(serialized_docs_dir):
for json_file in os.listdir(serialized_docs_dir):
if json_file.endswith(".json"):
doc = deserialize_document(json_file)
yield doc
# TODO: add to py-processors
def extract_from_documents(documents, rules, API):
for doc in documents:
mentions = API.odin.extract_from_document(doc, rules)
for m in sorted(mentions, key=lambda x: (x.document.id, x.sentence, x.start), reverse=False):
yield m
In [5]:
raw_text_files = [os.path.join(docs_dir, f) for f in os.listdir(docs_dir) if f.endswith(".txt")]
In [6]:
documents = [API.fastnlp.annotate(open(f).read()) for f in raw_text_files]
In [7]:
grammar_dir = os.path.join(demo_dir, "grammar")
my_prefix = "/Users/gus/repos/odin-tutorial"
master_file = open(os.path.join(grammar_dir, "master.yml"), "r").read().replace("/Users/gus/repos/odin-tutorial", my_prefix)
print(master_file)
In [8]:
print(open(os.path.join(grammar_dir, "events.yml"), "r").read())
In [9]:
mentions = list(extract_from_documents(documents, master_file, API))
In [10]:
display_mentions([m for m in mentions if m.matches("Event")])
In [11]:
viz.display_graph(documents[0].sentences[2])
In [12]:
example = """
# capturing locations
- name: "location-1"
label: Location
type: token
pattern: |
[lemma=/^(north|south|east|west)/]+ # starts with a direction word
[]{,2} # 0-2 intervening tokens
[lemma=axis]
- name: "location-2"
label: Location
type: token
pattern: |
[tag=/^NNP/ & incoming=prep_near] # proper noun w/ incoming "near" prep
("," [tag=NNP])*
- name: "location-3"
label: Location
type: token
pattern: |
[tag=/^NNP/]*
[tag=/^NNP/ & incoming=prep_at]
[tag=/^NNP/]*
"""
mns = list(extract_from_documents(documents, example, API))
display_mentions(mns)