In [56]:
import pandas as pd
import glob
from bs4 import BeautifulSoup
In [57]:
def store_data(check1,master_dict,tags,soup,email,user_df_list,keys,prin_pi,pal_count,usernames):
#store palmetto user count
if pal_count > 1:
palm_user = ','.join(usernames)
master_dict[keys[5]].append(palm_user)
elif pal_count == 1:
try:
master_dict[keys[5]].append(usernames[0])
except IndexError:
master_dict[keys[5]].append('Error')
pass
elif pal_count == 0:
master_dict[keys[5]].append('Not Found')
#store ID
awardID = soup.find(tags[0]).string
master_dict[keys[0]].append(awardID)
#Store Date
date = soup.find(tags[1]).string
master_dict[keys[1]].append(date)
#store cu count
master_dict[keys[2]].append(check1)
key = [x.string for x in soup.find_all(tags[2])]
values = [x.string for x in soup.find_all(tags[3])]
role_dict = dict(zip(key,values))
#store role check result
mail = ''
try:
for k,v in role_dict.items():
if v == prin_pi:
user = k.split('@')[0]
mail = k
if email in mail:
if user in user_df_list:
master_dict[keys[4]].append("Yes")
else:
master_dict[keys[4]].append("No")
else:
master_dict[keys[4]].append("No")
except KeyError:
master_dict[keys[4]].append("Error")
pass
In [58]:
%%time
prin_pi = 'Principal Investigator'
email = 'clemson.edu'
tagl = len(tags)
check = 0
pal_count = 0
usernames = []
tags = [
'AwardID',
'AwardEffectiveDate',
'EmailAddress',
'RoleCode'
]
master_dict = {
'AwardID':[],
'Date': [],
'CU_Email_Count':[],
'Palmetto_User_Count':[],
'Principal_Investigator':[],
'Palmetto_Username':[]
}
keys = [
'AwardID',
'Date',
'CU_Email_Count',
'Palmetto_User_Count',
'Principal_Investigator',
'Palmetto_Username'
]
#csv converted to list
df = pd.read_csv('../Research-Trend/palmetto/palmetto_data/Users.csv') #type = dataframe
user_df_Series = df['LocalUserId'] #type = Series, palmetto user ids
user_df_list = list(user_df_Series)
#list of awards
awards = glob.glob('20*/*.xml')
#parse awards
for docs in awards:
#open award
with open(docs) as award_doc:
xml = award_doc.read()
soup = BeautifulSoup(xml,'xml')
try:
for emails in soup.find_all(tags[2]):
at = emails.string #email address
us = emails.string.split('@')[0] #username
if email in at:
check+=1
if us in user_df_list:
usernames.append(us)
pal_count+=1 #number of palmetto users found
except AttributeError:
check = 0
continue
if check > 0:
check1 = check
check = 0
#store palmetto user count
master_dict[keys[3]].append(pal_count)
#store other variables
store_data(check1,master_dict,tags,soup,email,user_df_list,keys,
prin_pi,pal_count,usernames)
#reset values
check1 = 0
pal_count = 0
usernames = []
In [59]:
master_df = pd.DataFrame(dict([(k,pd.Series(v)) for k,v in master_dict.items()]))
m_df = master_df[['AwardID','Date','CU_Email_Count','Palmetto_User_Count',
'Principal_Investigator','Palmetto_Username']]
m_df
Out[59]:
In [60]:
check = m_df.loc[m_df['Palmetto_User_Count'] > 0]
check
Out[60]:
In [61]:
check2 = check.loc[m_df['Principal_Investigator'] == 'Yes']
check2
Out[61]: