In [1]:
%cd ~/NetBeansProjects/ExpLosion/
from notebooks.common_imports import *
from skll.metrics import kappa
from scipy.stats import spearmanr
from itertools import combinations
sns.timeseries.algo.bootstrap = my_bootstrap
sns.categorical.bootstrap = my_bootstrap
In [2]:
columns = 'Word 1,Word 2,Human (mean),1,2,3,4,5,6,7,8,9,10,11,12,13'.split(',')
df1 = pd.read_csv('../thesisgenerator/similarity-data/wordsim353/set1.csv')[columns]
df2 = pd.read_csv('../thesisgenerator/similarity-data/wordsim353/set2.csv')[columns]
df = pd.concat([df1, df2], ignore_index=True)
df_gold = pd.read_csv('../thesisgenerator/similarity-data/wordsim353/combined.csv',
names='w1 w2 sim'.split())
In [3]:
# had to remove trailing space from their files to make it parse with pandas
marco = pd.read_csv('../thesisgenerator/similarity-data/MEN/agreement/marcos-men-ratings.txt',
sep='\t', index_col=[0,1], names=['w1', 'w2', 'sim']).sort_index().convert_objects(convert_numeric=True)
elia = pd.read_csv('../thesisgenerator/similarity-data/MEN/agreement/elias-men-ratings.txt',
sep='\t', index_col=[0,1], names=['w1', 'w2', 'sim']).sort_index().convert_objects(convert_numeric=True)
In [4]:
df.head()
Out[4]:
In [5]:
# Each index ``i`` returned is such that ``bins[i-1] <= x < bins[i]``
def bin(arr, nbins=2, debug=False):
bins = np.linspace(arr.min(), arr.max(), nbins+1)
if debug:
print('bins are', bins)
return np.digitize(arr, bins[1:-1])
In [6]:
bin(df['1'], nbins=5, debug=True)[:10]
Out[6]:
In [7]:
bin(np.array([0, 2.1, 5.8, 7.9, 10]), debug=True) # 0 and 10 are needed to define the range of values
Out[7]:
In [8]:
bin(np.array([0, 2.1, 5.8, 7.9, 10]), nbins=3, debug=True)
Out[8]:
In [9]:
df.describe()
Out[9]:
In [10]:
elia.describe()
Out[10]:
In [11]:
bin_counts = range(2, 6)
# pair, bin count, kappa
kappas_pair = []
for name1, name2 in combinations(range(1,14), 2):
for b in bin_counts:
kappas_pair.append(['%d-%d'%(name1, name2),
b,
kappa(bin(df[str(name1)], b), bin(df[str(name2)], b))])
kappas_mean = []
for name in range(1, 14):
for b in bin_counts:
kappas_mean.append(['%d-m'%name,
b,
kappa(bin(df[str(name)], b), bin(df_gold.sim, b))])
kappas_men = [] # MEN data set- marco vs elia
for b in bin_counts:
kappas_men.append(['marco-elia',
b,
kappa(bin(marco.sim.values, b), bin(elia.sim.values, b))])
In [12]:
kappas1 = pd.DataFrame(kappas_pair, columns=['pair', 'bins', 'kappa'])
kappas1['kind'] = 'WS353-pairwise'
kappas2 = pd.DataFrame(kappas_mean, columns=['pair', 'bins', 'kappa'])
kappas2['kind'] = 'WS353-to mean'
kappas3 = pd.DataFrame(kappas_men, columns=['pair', 'bins', 'kappa'])
kappas3['kind'] = 'MEN'
kappas = pd.concat([kappas1, kappas2, kappas3], ignore_index=True)
kappas.head(3)
Out[12]:
In [13]:
with sns.color_palette("cubehelix", 3):
ax = sns.tsplot(kappas, time='bins', unit='pair', condition='kind', value='kappa',
marker='s', linewidth=4);
ax.set_xticklabels(np.arange(kappas.bins.min(), kappas.bins.max() + 0.01, 0.5).astype(np.int))
sparsify_axis_labels(ax)
plt.savefig('plot-intrinsic-ws353-kappas.pdf', format='pdf', dpi=300, bbox_inches='tight', pad_inches=0.1)
In [14]:
# sns.tsplot(kappas, time='bins', unit='pair', condition='kind', value='kappa',
sns.factorplot(data=kappas, x='bins', y='kappa', hue='kind', kind='box')
Out[14]:
In [15]:
kappas.groupby(['bins', 'kind']).mean()
Out[15]:
In [16]:
rhos_pair = []
for name1, name2 in combinations(range(1,14), 2):
rhos_pair.append(spearmanr(bin(df[str(name1)], b), bin(df[str(name2)], b))[0])
rhos_mean = []
for name in range(1,14):
rhos_mean.append(spearmanr(bin(df[str(name)], b), bin(df_gold.sim, b))[0])
In [17]:
sns.distplot(rhos_pair, label='pairwise');
# plt.axvline(np.mean(rhos_pair));
sns.distplot(rhos_mean, label='to mean');
# plt.axvline(np.mean(rhos_mean), color='g');
plt.legend(loc='upper left');
plt.savefig('plot-intrinsic-ws353-rhos.pdf', format='pdf', dpi=300, bbox_inches='tight', pad_inches=0.1)
print(np.mean(rhos_pair), np.mean(rhos_mean))
In [18]:
# Fisher transform: http://stats.stackexchange.com/a/19825 and wiki article therein
np.tanh(np.arctanh(rhos_pair).mean()), np.tanh(np.arctanh(rhos_mean).mean())
Out[18]:
In [19]:
from nltk.metrics.agreement import AnnotationTask
In [20]:
AnnotationTask(data=[
('coder1', 'obj1', 'label1'),
('coder1', 'obj2', 'label2'),
('coder2', 'obj1', 'label1'),
('coder2', 'obj2', 'label2'),
('coder3', 'obj1', 'label1'),
('coder3', 'obj2', 'label1'),
]).multi_kappa()
Out[20]:
In [21]:
multikappas = []
for name in range(1, 14):
for b in bin_counts:
labels = bin(df[str(name)], b)
# gold_labels = bin(df_gold.sim, b)
for i, label in enumerate(labels):
multikappas.append(('coder%d'%name, 'wordpair%d'%i, label))
In [22]:
AnnotationTask(multikappas).multi_kappa()
# WTF nltk, you are great
Out[22]:
In [23]:
spearmanr(marco.sim, elia.sim) # they report .6845
Out[23]:
In [ ]: