In [2]:
import numpy as np
import pandas as pd
import os
from sklearn import linear_model
from scipy import stats as stats
In [3]:
# load lb and test CIDs
# load LB CIDs
with open(os.path.abspath('__file__' + "/../../../data/CID_leaderboard.txt")) as f:
content = f.readlines()
lb_CIDs = list(content)
lb_CIDs = [int(x) for x in lb_CIDs]
# load test CIDs
with open(os.path.abspath('__file__' + "/../../../data/CID_testset.txt")) as f:
content = f.readlines()
test_CIDs = list(content)
test_CIDs = [int(x) for x in test_CIDs]
In [4]:
# load morgan matrix to use them as weights in training
morgan = pd.read_csv(os.path.abspath('__file__' + "/../../../data/morgan_sim.csv"), index_col=0)
weights = morgan[morgan.index.astype(str)]
weights = pd.concat((weights,weights)).sort()
print weights.shape
weights.head()
Out[4]:
In [5]:
#load the features
features = pd.read_csv('features.csv', index_col=0)
features.head()
Out[5]:
In [6]:
# give a number for each descriptor
descriptor = {}
for idx, desc in enumerate([u'INTENSITY/STRENGTH', u'VALENCE/PLEASANTNESS', u'BAKERY',
u'SWEET', u'FRUIT', u'FISH', u'GARLIC', u'SPICES', u'COLD', u'SOUR', u'BURNT',
u'ACID', u'WARM', u'MUSKY', u'SWEATY', u'AMMONIA/URINOUS', u'DECAYED', u'WOOD',
u'GRASS', u'FLOWER', u'CHEMICAL']):
descriptor[idx] = desc
In [7]:
# load the targets
all_targets = pd.read_csv('target.csv', index_col=0)
all_targets.head()
Out[7]:
In [8]:
#load the best feature numbers
best_feature_numbers = pd.read_csv('best_feature_numbers_CV.csv',index_col=0, header=None)
best_feature_numbers
Out[8]:
In [9]:
# predict LB
targets = all_targets[~all_targets['#oID'].isin(test_CIDs)]# remove test data
features = features[~features.CID.isin(test_CIDs)] # remove test data
train_targets = targets[~targets['#oID'].isin(lb_CIDs)] # exclude lb targets from training
train_features = features[~features.CID.isin(lb_CIDs)] # exclude lb features from training
test_features = features[features.CID.isin(lb_CIDs)]
# set the regressor
regr = linear_model.Ridge(alpha=1, fit_intercept=True, normalize=False, copy_X=True, max_iter=None, tol=0.001, solver='auto')
result = []
for idx in range(21):
print(descriptor[idx])
# load the scores for the descriptor
scores = pd.read_csv('LB_scores/scores_' + str(idx) + '.csv',index_col=0)
feature_number = int(best_feature_numbers.values[idx]) # set the number of best features to be used
X_all = train_features[scores.sort('0',ascending=0)[:feature_number].index] # set X values with the best features
X_all['CID'] = train_features.CID # add the CIDs as a column
for CID in lb_CIDs:
Y_train = train_targets[['#oID',descriptor[idx]]]
Y_train = Y_train[~Y_train[descriptor[idx]].isnull()]
X = X_all[X_all.CID.isin(Y_train['#oID'])]
weight = weights[weights.index.isin(Y_train['#oID'])][str(CID)]
regr.fit(X.drop('CID',1),Y_train[descriptor[idx]], sample_weight = weight.values)
if idx == 0: # if predicting intensity, use 1/1000 dilutions (neglog10 of 1/1000 is 3)
test_data = test_features[test_features.neglog10d == 3]
test_data = test_data[test_data.CID == CID]
test_data = test_data[scores.sort('0',ascending=0)[:feature_number].index]
else: # otherwise use high dilution data (not that they differ in this target matrix from the low ones)
test_data = test_features[test_features.Intensity == 1]
test_data = test_data[test_data.CID == CID]
test_data = test_data[scores.sort('0',ascending=0)[:feature_number].index]
Y_test = regr.predict(test_data)
std = -(Y_test**2)/2500.0+Y_test/25.0
result.append([CID, descriptor[idx], Y_test,std])
result = pd.DataFrame(result)
result.columns = ['#oID', 'descriptor', 'value', 'sigma']
In [10]:
result.value = result.value.astype(float)
result.sigma = result.sigma.astype(float)
# remove negative data and data above 100
result.value[result.value < 0] = 0
result.value[result.value > 100] = 100
result.sigma[result.sigma < 0] = 0
#result_mean['sigma'] = -(result_mean.value**2)/2500.0+result_mean.value/25.0
result.to_csv('subchallenge2.txt',sep='\t',index =0)
In [ ]: