In [ ]:

    

import datetime
import json
import six

print( "packages imported at " + str( datetime.datetime.now() ) )


    

In [ ]:

    

%pwd


    

In [ ]:

    

%run django_init.py


    

In [ ]:

    

# django imports
from django.contrib.auth.models import User

# sourcenet shared
from context_text.shared.person_details import PersonDetails

# sourcenet models.
from context_text.models import Article
from context_text.models import Article_Data
from context_text.models import Article_Subject
from context_text.models import Person
from context_text.shared.context_text_base import ContextTextBase
from context_text.tests.models.test_Article_Data_model import Article_Data_Copy_Tester

# sourcenet article_coding
from context_text.article_coding.article_coding import ArticleCoder
from context_text.article_coding.manual_coding.manual_article_coder import ManualArticleCoder

# context_analysis models.
from context_analysis.models import Reliability_Names
from context_analysis.reliability.reliability_names_builder import ReliabilityNamesBuilder

print( "sourcenet and context_analysis packages imported at " + str( datetime.datetime.now() ) )


    

In [ ]:

    

# folded this code into the Reliability_Names delete screen (context_analysis/views.py-->reliability_names_disagreement_view().
'''
reliability_names_id = "7956"
article_id = "21509"
article_data_id = "1660"
article_subject = "5498 (AS) - Jaidon ( id = 875; capture_method = OpenCalais_REST_API_v1 ) (mentioned; individual) ==> name: Jaidon"
    
markdown_string = "| "
markdown_string += reliability_names_id
markdown_string += " | Article ["
markdown_string += article_id
markdown_string += "](http://research.local/research/context/text/article/article_data/view_with_text/?article_id="
markdown_string += article_id
markdown_string += ") | Article_Data ["
markdown_string += article_data_id
markdown_string += "](http://research.local/research/context/text/article/article_data/view/?article_id="
markdown_string += article_id
markdown_string += "&article_data_id_select="
markdown_string += article_data_id
markdown_string += ") | "
markdown_string += article_subject
markdown_string += " |"

print( "Reliability_Names removal Markdown:\n" + markdown_string )
'''


    

In [ ]:

    

reliability_names_id_from = "9989"
reliability_names_id_to = "9988"
article_id = "23169"
article_data_id_from = "3195"
article_data_id_to = "2719"
article_subject = "12020 (AS) - Keller ( id = 2903; capture_method = OpenCalais_REST_API_v2 ) (quoted; individual) ==> name: Keller"

markdown_string = "| "
markdown_string += reliability_names_id_from
markdown_string += " | "
markdown_string += reliability_names_id_to
markdown_string += " | Article ["
markdown_string += article_id
markdown_string += "](http://research.local/research/context/text/article/article_data/view_with_text/?article_id="
markdown_string += article_id
markdown_string += ") | FROM ["
markdown_string += article_data_id_from
markdown_string += "](http://research.local/research/context/text/article/article_data/view/?article_id="
markdown_string += article_id
markdown_string += "&article_data_id_select="
markdown_string += article_data_id_from
markdown_string += ") TO ["
markdown_string += article_data_id_to
markdown_string += "](http://research.local/research/context/text/article/article_data/view/?article_id="
markdown_string += article_id
markdown_string += "&article_data_id_select="
markdown_string += article_data_id_to
markdown_string += ") | "
markdown_string += article_subject
markdown_string += " |"

print( "Reliabilty_Names merge Markdown:\n" + markdown_string )


    

In [ ]:

    

coder_id = 9
reliability_names_id = "9720"
article_id = "21130"
article_data_id = "2489"
article_subject = "8719 (AS) - Krueger ( id = 2015; capture_method = None ) (quoted; individual) ==> name: Krueger ==> organization: Ottawa County; Republican; Republican Party"
    
markdown_string = "| "
markdown_string += reliability_names_id
markdown_string += " | Article ["
markdown_string += article_id
markdown_string += "](http://research.local/research/context/text/article/article_data/view_with_text/?article_id="
markdown_string += article_id
markdown_string += ") | Article_Data ["
markdown_string += article_data_id
markdown_string += "](http://research.local/research/context/text/article/article_data/view/?article_id="
markdown_string += article_id
markdown_string += "&article_data_id_select="
markdown_string += article_data_id
markdown_string += ") | "
markdown_string += "Coder=" + str( coder_id )
markdown_string += " | "
markdown_string += article_subject
markdown_string += " |"

print( "Reliability_Names ground_truth fix Markdown:\n" + markdown_string )


    

In [ ]:

    

# declare variables
ground_truth_user = None
ground_truth_user_id = -1
id_of_article_data_to_copy = -1
new_article_data = None
new_article_data_id = -1
validation_error_list = None
validation_error_count = -1
validation_error = None

# set ID of article data we want to copy.
id_of_article_data_to_copy = 2489

# get the ground_truth user's ID.
ground_truth_user = ContextTextBase.get_ground_truth_coding_user()
ground_truth_user_id = ground_truth_user.id

# make the copy
new_article_data = Article_Data.make_deep_copy( id_of_article_data_to_copy,
                                                new_coder_user_id_IN = ground_truth_user_id )
new_article_data_id = new_article_data.id

# validate it.
validation_error_list = Article_Data_Copy_Tester.validate_article_data_deep_copy( original_article_data_id_IN = id_of_article_data_to_copy,
                                                                                  copy_article_data_id_IN = new_article_data_id,
                                                                                  copy_coder_user_id_IN = ground_truth_user_id )

# get error count:
validation_error_count = len( validation_error_list )
if ( validation_error_count > 0 ):
    
    # loop and output messages
    for validation_error in validation_error_list:
        
        print( "- Validation erorr: " + str( validation_error ) )
        
    #-- END loop over validation errors. --#
    
else:

    # no errors - success!
    print( "Record copy a success (as far as we know)!" )
    
#-- END check to see if validation errors --#

print( "copied Article_Data id " + str( id_of_article_data_to_copy ) + " INTO Article_Data id " + str( new_article_data_id ) + " at " + str( datetime.datetime.now() ) )


    

In [ ]:

    

# declare variables
article_data_id = -1
article_data = None
do_delete = False

# set ID.
article_data_id = 3314

# get model instance
article_data = Article_Data.objects.get( id = article_data_id )

# got something?
if ( article_data is not None ):
    
    # yes.  Delete?
    if ( do_delete == True ):
        
        # delete.
        print( "Deleting Article_Data: " + str( article_data ) )
        article_data.delete()
    
    else:
        
        # no delete.
        print( "Found Article_Data: " + str( article_data ) + ", but not deleting." )
        
    #-- END check to see if we delete --#
    
#-- END check to see if Article_Data match. --#


    

In [ ]:

    

# declare variables
article_id = -1
label = ""
do_delete = False
row_string_list = None

# first, get existing Reliability_Names rows for article and label.
article_id = 21130
label = "prelim_month"
#do_delete = True

# Do the delete
row_string_list = Reliability_Names.delete_reliabilty_names_for_article( article_id,
                                                                         label_IN = label,
                                                                         do_delete_IN = do_delete )

# print the strings.
for row_string in row_string_list:
    
    # print it.
    print( row_string )
    
#-- END loop over row strings --#


    

In [ ]:

    

# django imports
#from django.contrib.auth.models import User

# sourcenet imports
#from context_text.shared.context_text_base import ContextTextBase

# context_analysis imports
#from context_analysis.reliability.reliability_names_builder import ReliabilityNamesBuilder

# declare variables
my_reliability_instance = None
tag_in_list = []
article_id_in_list = []
label = ""

# declare variables - user setup
current_coder = None
current_coder_id = -1
current_index = -1

# declare variables - Article_Data filtering.
coder_type = ""

# make reliability instance
my_reliability_instance = ReliabilityNamesBuilder()

#===============================================================================
# configure
#===============================================================================

# list of tags of articles we want to process.
tag_in_list = [ "grp_month", ]

# list of IDs of articles we want to process:
article_id_in_list = [ 21130, ]

# label to associate with results, for subsequent lookup.
label = "prelim_month"

# ! ====> map coders to indices

# set it up so that...

# ...the ground truth user has highest priority (4) for index 1...
current_coder = ContextTextBase.get_ground_truth_coding_user()
current_coder_id = current_coder.id
current_index = 1
current_priority = 4
my_reliability_instance.add_coder_at_index( current_coder_id, current_index, priority_IN = current_priority )

# ...coder ID 8 is priority 3 for index 1...
current_coder_id = 8
current_index = 1
current_priority = 3
my_reliability_instance.add_coder_at_index( current_coder_id, current_index, priority_IN = current_priority )

# ...coder ID 9 is priority 2 for index 1...
current_coder_id = 9
current_index = 1
current_priority = 2
my_reliability_instance.add_coder_at_index( current_coder_id, current_index, priority_IN = current_priority )

# ...coder ID 10 is priority 1 for index 1...
current_coder_id = 10
current_index = 1
current_priority = 1
my_reliability_instance.add_coder_at_index( current_coder_id, current_index, priority_IN = current_priority )

# ...and automated coder (2) is index 2
current_coder = ContextTextBase.get_automated_coding_user()
current_coder_id = current_coder.id
current_index = 2
current_priority = 1
my_reliability_instance.add_coder_at_index( current_coder_id, current_index, priority_IN = current_priority )

# and only look at coding by those users.  And...

# configure so that it limits to automated coder_type of OpenCalais_REST_API_v2.
coder_type = "OpenCalais_REST_API_v2"
#my_reliability_instance.limit_to_automated_coder_type = "OpenCalais_REST_API_v2"
my_reliability_instance.automated_coder_type_include_list.append( coder_type )

# output debug JSON to file
#my_reliability_instance.debug_output_json_file_path = "/home/jonathanmorgan/" + label + ".json"

#===============================================================================
# process
#===============================================================================

# process articles
my_reliability_instance.process_articles( tag_in_list,
                                          article_id_in_list_IN = article_id_in_list )

# output to database.
my_reliability_instance.output_reliability_data( label )


    

In [ ]:

    

# initialization
test_manual_article_coder = None
test_person_details = None
test_article_subject = None
test_user = None

# create ManualArticleCoder
test_manual_article_coder = ManualArticleCoder()

# get test user
test_user = ContextTextBase.get_automated_coding_user()


    

In [ ]:

    

name_string = "Amber"
should_be = 0
test_qs = Person.find_person_from_name( name_string )
match_count = test_qs.count()
error_string = "==> Person.find_person_from_name( " + name_string + " ) --> " + str( match_count ) + " should = " + str( should_be )
print( error_string )

person_counter = 0
for person_instance in test_qs:
    
    person_counter += 1
    print( "Person " + str( person_counter ) + ": " + str( person_instance ) )
    
    
#-- END loop over person matches --#


    

In [ ]:

    

# test person values
lookup_person_id = -1
lookup_person_name = "Amber"
lookup_title = ""

# get ManualArticleCoder instance - in init above
#test_manual_article_coder = ManualArticleCoder()

# set up person details
test_person_details = PersonDetails()
test_person_details[ ArticleCoder.PARAM_PERSON_NAME ] = lookup_person_name
#test_person_details[ ArticleCoder.PARAM_PERSON_ID ] = lookup_person_id
#test_person_details[ ArticleCoder.PARAM_TITLE ] = lookup_title

# make test Article_Subject
test_article_subject = Article_Subject()

# lookup person - returns person and confidence score inside
#    Article_Subject instance.
test_article_subject = test_manual_article_coder.lookup_person( test_article_subject, 
                                                                lookup_person_name,
                                                                create_if_no_match_IN = False,
                                                                update_person_IN = False,
                                                                person_details_IN = test_person_details )

# get results from Article_Subject
test_person = test_article_subject.person
test_person_match_list = test_article_subject.person_match_list  # list of Person instances

# output results
print( "==> ManualArticleCoder.lookup_person() result for name \"" + lookup_person_name + "\": " + str( test_person ) )
print( "ManualArticleCoder.lookup_person() match list: " + str( test_person_match_list ) )


    

In [ ]:

    

# create ManualArticleCoder instance.
#test_manual_article_coder = ManualArticleCoder()

# get an article.
test_article = Article.objects.get( pk = 22874 )

# create bare-bones Article_Data
test_article_data = Article_Data()
test_article_data.coder = test_user
test_article_data.article = test_article
test_article_data.save()

#----------------------------------------------------------------------#
# !test 1 - with person ID.       
#----------------------------------------------------------------------#

# retrieve person information.
person_name = "Amber"
title = ""
person_id = -1

# set up person details
person_details = PersonDetails()
person_details[ ManualArticleCoder.PARAM_PERSON_NAME ] = person_name
person_details[ ManualArticleCoder.PARAM_NEWSPAPER_INSTANCE ] = test_article.newspaper

# got a title?
if ( ( title is not None ) and ( title != "" ) ):

    # we do.  store it in person_details.
    person_details[ ManualArticleCoder.PARAM_TITLE ] = title

#-- END check to see if title --#

# got a person ID?
if ( ( person_id is not None ) and ( person_id != "" ) and ( person_id > 0 ) ):

    # we do.  store it in person_details.
    person_details[ ManualArticleCoder.PARAM_PERSON_ID ] = person_id

#-- END check to see if title --#

# create an article_subject.
test_article_subject = test_manual_article_coder.process_subject_name( test_article_data, person_name, person_details_IN = person_details )

# check to make sure not None
status_message = "==> ArticleCoder.process_subject_name() for name \"" + str( person_name ) + "\": " + str( test_article_subject )
print( status_message )


    

In [ ]:

    

name_string = "Logan"
should_be = 0
test_qs = Person.find_person_from_name( name_string )
match_count = test_qs.count()
error_string = "==> Person.find_person_from_name( " + name_string + " ) --> " + str( match_count ) + " should = " + str( should_be )
print( error_string )

person_counter = 0
for person_instance in test_qs:
    
    person_counter += 1
    print( "Person " + str( person_counter ) + ": " + str( person_instance ) )
    
    
#-- END loop over person matches --#


    

In [ ]:

    

# test person values
lookup_person_id = -1
lookup_person_name = "Logan"
lookup_title = ""

# get ManualArticleCoder instance
test_manual_article_coder = ManualArticleCoder()

# set up person details
test_person_details = PersonDetails()
test_person_details[ ArticleCoder.PARAM_PERSON_NAME ] = lookup_person_name
#test_person_details[ ArticleCoder.PARAM_PERSON_ID ] = lookup_person_id
#test_person_details[ ArticleCoder.PARAM_TITLE ] = lookup_title

# make test Article_Subject
test_article_subject = Article_Subject()

# lookup person - returns person and confidence score inside
#    Article_Subject instance.
test_article_subject = test_manual_article_coder.lookup_person( test_article_subject, 
                                                                lookup_person_name,
                                                                create_if_no_match_IN = False,
                                                                update_person_IN = False,
                                                                person_details_IN = test_person_details )

# get results from Article_Subject
test_person = test_article_subject.person
test_person_match_list = test_article_subject.person_match_list  # list of Person instances

# output results
print( "==> ManualArticleCoder.lookup_person() result for name \"" + lookup_person_name + "\": " + str( test_person ) )
print( "ManualArticleCoder.lookup_person() match list: " + str( test_person_match_list ) )


    

In [ ]:

    

# create ManualArticleCoder instance.
#test_manual_article_coder = ManualArticleCoder()

# get an article.
test_article = Article.objects.get( pk = 22970 )

# create bare-bones Article_Data
test_article_data = Article_Data()
test_article_data.coder = test_user
test_article_data.article = test_article
test_article_data.save()

#----------------------------------------------------------------------#
# !test 1 - with person ID.       
#----------------------------------------------------------------------#

# retrieve person information.
person_name = "Logan"
title = ""
person_id = -1

# set up person details
person_details = PersonDetails()
person_details[ ManualArticleCoder.PARAM_PERSON_NAME ] = person_name
person_details[ ManualArticleCoder.PARAM_NEWSPAPER_INSTANCE ] = test_article.newspaper

# got a title?
if ( ( title is not None ) and ( title != "" ) ):

    # we do.  store it in person_details.
    person_details[ ManualArticleCoder.PARAM_TITLE ] = title

#-- END check to see if title --#

# got a person ID?
if ( ( person_id is not None ) and ( person_id != "" ) and ( person_id > 0 ) ):

    # we do.  store it in person_details.
    person_details[ ManualArticleCoder.PARAM_PERSON_ID ] = person_id

#-- END check to see if title --#

# create an article_subject.
test_article_subject = test_manual_article_coder.process_subject_name( test_article_data, person_name, person_details_IN = person_details )

# check to make sure not None
status_message = "==> ArticleCoder.process_subject_name() for name \"" + str( person_name ) + "\": " + str( test_article_subject )
print( status_message )


    

In [ ]:

    

# load test data fixture JSON into memory.
fixture_directory_path = '/home/jonathanmorgan/work/django/research/context/text/fixtures'
test_data_fixture_file_name = 'sourcenet_unittest_data.json'
fixture_file_path = fixture_directory_path + "/" + test_data_fixture_file_name

# constants-ish
person_model_name = "sourcenet.person"
json_name_model = "model"
json_name_fields = "fields"
json_name_field_first_name = "first_name"
json_name_field_last_name = "last_name"

# tracking counts of names
model_name_to_count_map = {}
model_none_count = -1
record_count = -1
person_count = -1
first_name_to_info_map = {}
first_name_to_count_map = {}
data_record = None
person_info = None
person_fields = None
model = ""
first_name = ""
first_name_count = -1
first_name_none_count = -1
last_name = ""

# read file and use json to load it.
with open( fixture_file_path, 'r' ) as fixture_file:
    
    # load into memory and parse.
    fixture_data = json.load( fixture_file )

#-- END with open( fixture_file_path )... --#
    
# how many things?
print( "There are " + str( len( fixture_data ) ) + " items in fixture " + fixture_file_path + " - type: " + str( type( fixture_data ) ) )

# loop over records.
model_none_count = 0
record_count = 0
person_count = 0
person_info = None
first_name_none_count = 0
first_name_empty_count = 0
for data_record in fixture_data:

    record_count += 1

    # First, check to see if type is sourcenet.person
    model = data_record.get( json_name_model, None )
    if ( ( model != None ) and ( model == person_model_name ) ):

        if model not in model_name_to_count_map:

            # add to count map with count set to 1.
            model_name_to_count_map[ model ] = 1

        else:

            # increment counter
            model_count = model_name_to_count_map[ model ]
            model_count += 1
            model_name_to_count_map[ model ] = model_count

        #-- END check to see if model in model_name_set. --#

        # it is a person.
        person_count += 1
        person_info = data_record
        person_fields = person_info.get( json_name_fields, None )

        # make sure we have fields
        if ( person_fields is not None ):

            # we have fields.

            # get first name
            first_name = person_fields.get( json_name_field_first_name, None )
            #print( "First name string = " + str( first_name ) )
            last_name = person_fields.get( json_name_field_last_name, None )

            # Got one?
            if ( first_name is not None ):

                # ...that is not empty
                first_name = first_name.strip()
                if ( first_name != "" ):

                    # yes.  Check for first name in count map.
                    if ( first_name in first_name_to_count_map ):

                        # it is there.  Increment count and clear out person_info...
                        first_name_count = first_name_to_count_map[ first_name ]
                        first_name_count += 1
                        first_name_to_count_map[ first_name ] = first_name_count

                        # ...and clear out the person info.
                        first_name_to_info_map[ first_name ] = None


                    else:

                        # it is not there.  Set count to 1 and store person_info.
                        first_name_count = 1
                        first_name_to_count_map[ first_name ] = first_name_count
                        first_name_to_info_map[ first_name ] = person_info

                    #-- END check to see if first name. --#

                else:
                
                    # increment first_name_empty_count --#
                    first_name_empty_count += 1
                
                #-- END check for first name not empty. --#

            else:
                
                # first name is None.
                first_name_none_count += 1
                
            #-- END check for first name. --#

        else:

            print( "ERROR - no fields in " + str( person_info ) )

        #-- END check for fields --#
        
    else:
        
        if ( model is not None ):
            
            if model not in model_name_to_count_map:
                
                model_name_to_count_map[ model ] = 1
                
            else:
                
                model_count = -1
                model_count = model_name_to_count_map[ model ]
                model_count += 1
                model_name_to_count_map[ model ] = model_count
                
            #-- END check to see if model in model_name_set. --#
            
        else:
            
            model_none_count += 1
            
        #-- END check to see if model is None --#

    #-- END check to see if sourcenet.person --#

#-- END loop over data records. --#

print( "Found " + str( len( first_name_to_count_map ) ) + " sourcenet.person records out of " + str( record_count ) + " records." )
print( "first names to counts: " + str( first_name_to_count_map ) )
print( "==> Model name set = " + str( model_name_to_count_map ) + "; model_none_count = " + str( model_none_count ) )
print( "==> first_name_none_count = " + str( first_name_none_count ) )
print( "==> first_name_empty_count = " + str( first_name_empty_count ) )


    

In [ ]:

    

selected_person = first_name_to_info_map.get( 'Anirban', None )
print( "Selected person: " )
print( str( selected_person ) )