In [1]:
# This tells matplotlib not to try opening a new window for each plot.
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
# General libraries.
import json
import re
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import datetime
# SK-learn libraries for learning.
from sklearn.pipeline import Pipeline
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import GridSearchCV #update module model_selection
from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
# SK-learn libraries for evaluation.
from sklearn.metrics import confusion_matrix
from sklearn import metrics
from sklearn.decomposition import PCA
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import cross_val_predict, cross_val_score
from sklearn import preprocessing
from sklearn.mixture import GMM
# SK-learn libraries for feature extraction from text.
from sklearn.feature_extraction.text import *
In [2]:
#load json training data into pandas dataframe
df = pd.read_json('train.json')
df.info()
In [3]:
df['requester_received_pizza'] = np.where(df['requester_received_pizza'] == True, 1, 0)
df['requester_received_pizza'].value_counts()
Out[3]:
In [4]:
good_indexes = []
for i, name in enumerate(df.columns):
if re.findall('retrieval', name):
pass
else:
good_indexes.append(i)
# Remove at_retrieval fields from dataframce df
columns = df.columns[good_indexes]
df = df.loc[:,columns]
In [5]:
#Drop six more columns from dataset
df.drop(['giver_username_if_known', 'post_was_edited', 'request_id',
'requester_user_flair', 'requester_username'], axis=1, inplace=True)
df.info()
In [6]:
#Show that 104 observations have a blank "request_text" field
len(df[df['request_text'].str.len() == 0])
Out[6]:
In [7]:
#1. Combine request_text and request_title fields
#2. Lowercase all words
df['request_text_n_title'] = (df['request_title'] + ' ' + df['request_text_edit_aware'])
df['request_text_n_title'] = [ text.split(" ",1)[1].lower() for text in df['request_text_n_title']]
print df['request_text_n_title'].head()
#3. Add a total length feature to the dataset
df['total_length'] =df['request_text_n_title'].apply(lambda x: len(x.split(' ')))
#4. Ensure there are no zero length requests in the new feature/column
print '\nAfter combining request_title and request_text, number of requests with length of \
"zero": {}'.format(len(df[df['request_text_n_title'] == 0]))
In [8]:
#Showcase of new added features to dataset
df.loc[:5,['request_text_n_title', 'total_length']]
Out[8]:
In [9]:
np.random.seed(0)
#separate into features and labels
text_features = df['request_text_n_title'].values # text features
success_rate = sum(df['requester_received_pizza'])/4040.
print "Original success rate: {}".format(round(success_rate, 4))
# Create target field for received pizza are only 1's and 0's
target = df['requester_received_pizza'].values
#shuffle our data to ensure randomization
shuffle = np.random.permutation(len(text_features))
text_features, target = text_features[shuffle], target[shuffle]
#separate into training and dev groups
train_data, train_labels = text_features[:3200], target[:3200]
dev_data, dev_labels = text_features[3200:], target[3200:]
#check to ensure success rate is roughly preserved across sets
train_success_rate = sum(train_labels)/3200.
dev_success_rate = sum(dev_labels)/840.
print "Training success rate: {}".format(round(train_success_rate, 4))
print "Dev success rate: {}".format(round(dev_success_rate, 4))
#check to ensure we've got the right datasets
print '\n\nTraining Data shape: \t{}'.format(train_data.shape)
print 'Training Labels shape: \t{}'.format(train_labels.shape)
print 'Dev data shape: \t{}'.format(dev_data.shape)
print 'Dev Labels shape: \t{}'.format(dev_labels.shape)
In [10]:
#fit logistic classifier to training data
print "BASELINE LOGISTIC REGRESSION"
print "----------------------------"
vec = TfidfVectorizer()
train_matrix = vec.fit_transform(train_data)
dev_matrix = vec.transform(dev_data)
lr= LogisticRegression(n_jobs=-1, class_weight='balanced').fit(train_matrix, train_labels)
predictions = lr.predict(dev_matrix)
score = round(roc_auc_score(dev_labels, predictions, average='weighted'), 4)
print "Baseline ROC AUC score: {}".format(score)
print "\n\nRESTRICTED FEATURES LOGISTIC REGRESSION"
print "---------------------------------------"
model_scores = []
range_features = np.arange(385,400)
for features in range_features:
vec = TfidfVectorizer(stop_words='english',sublinear_tf=1, ngram_range=(1,1),max_features=features)
train_matrix = vec.fit_transform(train_data)
dev_matrix = vec.transform(dev_data)
lr= LogisticRegression(n_jobs=-1, class_weight='balanced').fit(train_matrix, train_labels)
predictions = lr.predict(dev_matrix)
model_scores.append(round(metrics.roc_auc_score(dev_labels, predictions, average = 'weighted'), 4))
best_score = round(max(model_scores), 4)
print "Max number of features: {}".format(range_features[np.argmax(model_scores)])
print "Best ROC AUC Score: {}".format(best_score)
#best_max_feature
best_max_feature = range_features[np.argmax(model_scores)]
#fit logistic classifier to training data
vec = TfidfVectorizer(stop_words='english',sublinear_tf=1, ngram_range=(1,1), max_features=best_max_feature)
train_matrix = vec.fit_transform(train_data)
dev_matrix = vec.transform(dev_data)
print "\n"
print "LOGISTIC REGRESSION with tuning c value and restricted # of features"
print "--------------------------------------------------------------------"
c_values = np.logspace(.0001, 2, 200)
c_scores = []
for value in c_values:
lr = LogisticRegression(C=value, n_jobs=-1, class_weight='balanced', penalty='l2')
lr.fit(train_matrix, train_labels)
predictions = lr.predict(dev_matrix)
c_scores.append(round(metrics.roc_auc_score(dev_labels, predictions, average = 'weighted'), 4))
best_c_value = c_values[np.argmax(c_scores)]
print "Best C-value: {}".format(best_c_value)
print "Best ROC AUC Score: {}".format(c_scores[np.argmax(c_scores)])
lr = LogisticRegression(C=best_c_value, n_jobs=-1).fit(train_matrix, train_labels)
predictions = lr.predict(dev_matrix)
print '\n'
In [11]:
#fit Multinomial Naive Bayes classifier to training data
print "BERNOULLI NAIVE BAYES"
print "-----------------------"
model_scores = []
range_features = np.arange(415, 431)
for features in range_features:
vec = TfidfVectorizer(max_features=features)
train_matrix = vec.fit_transform(train_data)
dev_matrix = vec.transform(dev_data)
alphas = np.linspace(.0001, 1, 20)
bnb = BernoulliNB(alpha=.001).fit(train_matrix, train_labels)
predictions = bnb.predict(dev_matrix)
model_scores.append(round(roc_auc_score(dev_labels, predictions, average='weighted'), 4))
print "Best ROC AUC Score: {}".format(max(model_scores))
print "Max Features: {}".format(range_features[np.argmax(model_scores)])
In [12]:
print "\nSupport Vector Machine"
print "------------------------"
vec = TfidfVectorizer(max_features=best_max_feature, stop_words='english')
train_matrix = vec.fit_transform(train_data)
dev_matrix = vec.transform(dev_data)
svc = LinearSVC().fit(train_matrix, train_labels)
predictions = svc.predict(dev_matrix)
print round(roc_auc_score(dev_labels, predictions, average='weighted'),4)
In [13]:
#Create feature where in image is included in the request
df['image_incl'] = np.where(df['request_text_n_title'].str.contains("imgur"), int(1), int(0))
In [14]:
#Create feature that is an aggregate of all indicators of community status including seniority
df['karma'] = df['requester_account_age_in_days_at_request'] + df['requester_days_since_first_post_on_raop_at_request']\
+ df['requester_number_of_comments_at_request'] + df['requester_number_of_comments_in_raop_at_request'] + \
df['requester_number_of_posts_at_request'] + df['requester_number_of_posts_on_raop_at_request'] + \
df['requester_number_of_subreddits_at_request'] + df['requester_upvotes_minus_downvotes_at_request']
karma_winners = df['karma'][df['requester_received_pizza'] == 1].describe()
karma_losers = df['karma'][df['requester_received_pizza'] == 0].describe()
karma_comparison = pd.concat([karma_winners, karma_losers], axis=1)
karma_comparison.columns = ['winners', 'losers']
karma_comparison
Out[14]:
In [15]:
low_losers= len(df[df['requester_received_pizza'] == 0][df['karma'] < 15])
low_winners=len(df[df['requester_received_pizza'] == 1][df['karma'] < 15])
print low_losers, low_losers/3046.
print low_winners, low_winners/994.
high_losers = len(df[df['requester_received_pizza'] == 0][df['karma'] > 5000])
high_winners = len(df[df['requester_received_pizza'] == 1][df['karma'] > 5000])
print high_losers, high_losers/3046.
print high_winners, high_winners/994.
df['karma_low'] = np.where(df['karma'] < 15, 1, 0)
In [16]:
length_winners = df['total_length'][df['requester_received_pizza'] == 1].describe()
length_losers = df['total_length'][df['requester_received_pizza'] == 0].describe()
length_comparison = pd.concat([length_winners, length_losers], axis=1)
length_comparison.columns = ['winners', 'losers']
length_comparison
Out[16]:
In [17]:
df['day']=df['unix_timestamp_of_request_utc'].apply(lambda x:int(datetime.datetime.fromtimestamp(int(x)).strftime('%d')))
df['time']=df['unix_timestamp_of_request'].apply(lambda x:int(datetime.datetime.fromtimestamp(int(x)).strftime('%H')))
df['first_half'] = np.where(df['day'] < 16, 1, 0)
In [18]:
#Create binary variables that show whether requester is grateful or willing to "pay it forward"
df['requester_grateful'] = np.where(df['request_text_n_title'].str.contains('thanks' or 'advance' or 'guy'\
or 'reading' or 'anyone' or 'anything' or'story'or 'tonight'or 'favor'or'craving'), int(1), int(0))
df['requester_payback'] = np.where(df['request_text_n_title'].str.contains('return' or 'pay it back'\
or 'pay it forward' or 'favor'), int(1), int(0))
In [19]:
# Define narrative categories
narratives = {
'money':['money','now','broke','week','until','time',
'last','day','when','today','tonight','paid',
'next','first','night','night','after','tomorrow',
'while','account','before','long','friday','rent',
'buy','bank','still','bills','ago','cash','due',
'soon','past','never','paycheck','check','spent',
'year','years','poor','till','yesterday','morning',
'dollars','financial','hour','bill','evening','credit',
'budget','loan','bucks','deposit','dollar','current','payed'],
'job':['work','job','paycheck','unemployment','interviewed',
'fired','employment','hired','hire'],
'student':['college','student','school','roommate','studying',
'study','university','finals','semester','class','project',
'dorm','tuition'],
'family': ['family','mom','wife','parents','mother','husband','dad','son',
'daughter','father','parent','mum','children','starving','hungry'],
'craving': ['friend','girlfriend','birthday','boyfriend','celebrate',
'party','game','games','movie','movies','date','drunk',
'beer','celebrating','invited','drinks','crave','wasted','invited']
}
# function to extract word count for each narrative from one text post, normalize by the word count of that post
def single_extract(text):
count = {'money':0.,
'job':0.,
'student':0.,
'family':0.,
'craving':0.}
words = text.split(' ')
length = 1./len(words)
for word in text.split(' '):
for i,k in narratives.items():
if word in k:
count[i] += length
return count.values()
# Extract request_text_n_title field
texts = df['request_text_n_title'].copy()
#initialize count
count =[]
# return normalized count for each narrative from all requests
for text in texts:
count.append(single_extract(text))
# narrative dataframe
narrative = pd.DataFrame(count)
# set up median for using with the test set
median_values = []
#extract narrative field
for i,k in enumerate(narratives.keys()):
median_values.append(np.median(narrative[i]))
narrative['narrative_'+k] = (narrative[i] > np.median(narrative[i])).astype(int)
narrative.drop([i],axis=1,inplace=True)
# concatenate
df = pd.concat([df,narrative],axis=1)
In [20]:
df.info()
In [21]:
continuous_list = ['total_length']#, 'time', 'karma']
binary_list = [
u'image_incl',
u'karma_low',
# u'month',
# u'day',
u'time',
u'first_half',
u'requester_grateful',
u'requester_payback',
u'narrative_money',
u'narrative_job',
u'narrative_family',
u'narrative_student',
u'narrative_craving'
]
#create new DataFrame using previously defined "numeric_features" object to determine all columns in the DF
numeric_features = df.copy().loc[:,continuous_list]
numeric_features_norm = pd.DataFrame(data=preprocessing.normalize(numeric_features, axis=0),\
columns=numeric_features.columns.values)
# combine to contious and binary
numeric_features_norm = pd.concat([df[binary_list],numeric_features_norm],axis=1)
numeric_features_norm.head()
Out[21]:
In [22]:
features = numeric_features_norm.copy()
target = df['requester_received_pizza']
features.info()
In [23]:
X, y = features.values, target.copy()
#shuffle = np.random.permutation(len(X))
X, y = X[shuffle], y[shuffle]
Xtrain, Xdev, ytrain, ydev = X[:3200], X[3200:], y[:3200], y[3200:]
Xtrain.shape, Xdev.shape, ytrain.shape, ydev.shape
Out[23]:
In [24]:
scores = []
c_values = np.linspace(.001, 100, 20)
for c in c_values:
lr_8 = LogisticRegression(C=c, class_weight='balanced', n_jobs=-1).fit(Xtrain, ytrain)
predictions = lr_8.predict(Xdev)
scores.append(round(roc_auc_score(ydev, predictions, average='weighted'), 4))
print "Best C-value: {}".format(c_values[np.argmax(scores)])
print 'Best AUC score based on metrics.roc_auc_score = {}'.format(max(scores))
In [25]:
#Initialize our vectorizer using hyperparameters from previous models
vec = TfidfVectorizer(stop_words='english',sublinear_tf=1, max_features=best_max_feature)
train_matrix = vec.fit_transform(train_data)
#Transform our text features
text_features = df['request_text_n_title'].values.copy()
text_transformed = vec.transform(text_features)
# Concatenate egineered numeric_features with vectorized text features
combined_features = np.append(features.values, text_transformed.toarray(), axis = 1)
print combined_features.shape
#prep data for modeling
X, y = combined_features, target.copy()
X, y = X[shuffle], y[shuffle]
Xtrain, Xdev, ytrain, ydev = X[:3200], X[3200:], y[:3200], y[3200:]
Xtrain.shape, Xdev.shape, ytrain.shape, ydev.shape
#Fit logistic model to data
c_values = np.logspace(.0001, 2, 200)
c_scores = []
for value in c_values:
lr = LogisticRegression(C=value, n_jobs=-1, class_weight='balanced', penalty='l2')
lr.fit(Xtrain, ytrain)
predictions = lr.predict(Xdev)
c_scores.append(round(metrics.roc_auc_score(ydev, predictions, average = 'weighted'), 4))
best_C_value = c_values[np.argmax(c_scores)]
print "Best C-value: {}".format(best_C_value)
print "Best ROC AUC Score: {}".format(c_scores[np.argmax(c_scores)])
In [26]:
#fit logistic classifier to training data
vec = TfidfVectorizer(stop_words='english',sublinear_tf=1, max_features=best_max_feature)
train_matrix = vec.fit_transform(train_data)
lr1 = LogisticRegression(C=best_c_value, n_jobs=-1).fit(train_matrix, train_labels)
text_features = df['request_text_n_title'].values.copy()
text_transformed = vec.transform(text_features)
# create pizza_predict field for all records based on features array
pizza_predict = lr1.predict_proba(text_transformed)[:,1][:, np.newaxis]
# Concatenate numeric_features with pizza_predict to create pizza_predict + numeric features in ens_features
combined_features = np.append(features.values, pizza_predict, axis = 1)
X, y = combined_features, target.copy()
#shuffle = np.random.permutation(len(X))
X, y = X[shuffle], y[shuffle]
Xtrain, Xdev, ytrain, ydev = X[:3200], X[3200:], y[:3200], y[3200:]
Xtrain.shape, Xdev.shape, ytrain.shape, ydev.shape
#####
c_values = np.logspace(.0001, 2, 200)
c_scores = []
for value in c_values:
lr = LogisticRegression(C=value, n_jobs=-1, class_weight='balanced', penalty='l2')
lr.fit(Xtrain, ytrain)
predictions = lr.predict(Xdev)
c_scores.append(round(metrics.roc_auc_score(ydev, predictions, average = 'weighted'), 4))
best_C_value = c_values[np.argmax(c_scores)]
print "Best C-value: {}".format(best_C_value)
print "Best ROC AUC Score: {}".format(c_scores[np.argmax(c_scores)])
In [27]:
"""Our first attempt is a brute force method with logistics regression that
use the best parameters from developing the model"""
df_test = pd.read_json('test.json')
df_test['request_text_n_title'] = (df_test['request_title'] + ' ' + df_test['request_text_edit_aware'])
df_test['request_text_n_title'] = [ text.split(" ",1)[1].lower() for text in df_test['request_text_n_title']]
# print df_test['request_text_n_title'].head()
#create test set
test_features = df_test['request_text_n_title'].values # text features
#create new training set using all 4040 training samples
train_features = df['request_text_n_title'].values
train_labels = df['requester_received_pizza'].values
#shuffle our data to ensure randomization
shuffle = np.random.permutation(len(train_features))
train_features, train_labels = train_features[shuffle], train_labels[shuffle]
#transform raw data into Tfdif vector
vec = TfidfVectorizer(stop_words='english',sublinear_tf=1, max_features=387)
train_matrix = vec.fit_transform(train_features)
test_matrix = vec.transform(test_features)
#fit Logistic Regression model
lr= LogisticRegression(n_jobs=-1, class_weight='balanced').fit(train_matrix, train_labels)
test_predictions = lr.predict(test_matrix)[:, np.newaxis]
# print type(predictions_test)
sub_1 = np.append(df_test['request_id'].values[:, np.newaxis], test_predictions, axis = 1)
sub_1_df = pd.DataFrame(data=sub_1, columns=['request_id', 'requester_received_pizza']) # 1st row as the column names
sub_1_df.to_csv("submission_1.csv", sep=',', header=True, mode='w', index=0)
"""With this submission, we get a AUC score of 0.59386"""
print "Kaggle submission 1, result AUC = 0.59386"
In [28]:
"""The 2nd submission, we define a function process
to process the train and test json files
This function use the file name and the max number of feature tuned previously.
The output of process function depends on if it's train or test set
"""
def process(filename,max_feature_length,train=True):
df = pd.read_json(filename)
df['request_text_n_title'] = (df['request_title'] + ' ' + df['request_text_edit_aware'])
df['request_text_n_title'] = [ text.split(" ",1)[1].lower() for text in df['request_text_n_title']]
#total length
df['total_length'] =df['request_text_n_title'].apply(lambda x: len(x.split(' ')))
#get engineer features
# image included?
df['image_incl'] = np.where(df['request_text_n_title'].str.contains("imgur"), int(1), int(0))
#Karma
df['karma'] = df['requester_account_age_in_days_at_request']\
+ df['requester_days_since_first_post_on_raop_at_request']\
+ df['requester_number_of_comments_at_request'] + df['requester_number_of_comments_in_raop_at_request'] + \
df['requester_number_of_posts_at_request'] + df['requester_number_of_posts_on_raop_at_request'] + \
df['requester_number_of_subreddits_at_request'] + df['requester_upvotes_minus_downvotes_at_request']
df['karma_low'] = np.where(df['karma'] < 15, 1, 0)
# time
df['day']=df['unix_timestamp_of_request_utc'].apply(lambda x:int(datetime.datetime.fromtimestamp(int(x)).strftime('%d')))
df['time']=df['unix_timestamp_of_request'].apply(lambda x:int(datetime.datetime.fromtimestamp(int(x)).strftime('%H')))
df['first_half'] = np.where(df['day'] < 16, 1, 0)
# requester's attitude
df['requester_grateful'] = np.where(df['request_text_n_title'].
str.contains('thanks' or 'advance' or 'guy'\
or 'reading' or 'anyone' or 'anything'\
'story'or 'tonight'or 'favor'or'craving'),
int(1), int(0))
df['requester_payback'] = np.where(df['request_text_n_title'].
str.contains('return' or 'pay it back' or 'pay it forward' or 'favor'), int(1), int(0))
#narrative
# Define narrative categories
narratives = {
'money':['money','now','broke','week','until','time',
'last','day','when','today','tonight','paid',
'next','first','night','night','after','tomorrow',
'while','account','before','long','friday','rent',
'buy','bank','still','bills','ago','cash','due',
'soon','past','never','paycheck','check','spent',
'year','years','poor','till','yesterday','morning',
'dollars','financial','hour','bill','evening','credit',
'budget','loan','bucks','deposit','dollar','current','payed'],
'job':['work','job','paycheck','unemployment','interviewed',
'fired','employment','hired','hire'],
'student':['college','student','school','roommate','studying',
'study','university','finals','semester','class','project',
'dorm','tuition'],
'family': ['family','mom','wife','parents','mother','husband','dad','son',
'daughter','father','parent','mum','children','starving','hungry'],
'craving': ['friend','girlfriend','birthday','boyfriend','celebrate',
'party','game','games','movie','movies','date','drunk',
'beer','celebrating','invited','drinks','crave','wasted','invited']
}
# function to extract word count for each narrative from one text post
# normalize by the word count of that post
def single_extract(text):
count = {'money':0.,
'job':0.,
'student':0.,
'family':0.,
'craving':0.}
words = text.split(' ')
length = 1./len(words)
for word in text.split(' '):
for i,k in narratives.items():
if word in k:
count[i] += length
return count.values()
# Extract request_text_n_title field
texts = df['request_text_n_title'].copy()
#initialize count
count =[]
# return normalized count for each narrative from all requests
for text in texts:
count.append(single_extract(text))
# narrative dataframe
narrative = pd.DataFrame(count)
# set up median for using with the test set
#extract narrative field
for i,k in enumerate(narratives.keys()):
narrative['narrative_'+k] = (narrative[i] > np.median(narrative[i])).astype(int)
narrative.drop([i],axis=1,inplace=True)
# concatenate
df = pd.concat([df,narrative],axis=1)
continuous_list = ['total_length','time']
# include relevant binary variables and avoid overfitting
binary_list = ['requester_grateful','narrative_job',
'narrative_family', 'narrative_craving',
'image_incl']
#create new DataFrame using previously defined "numeric_features" object to determine all columns in the DF
continuous_features = df.copy().loc[:,continuous_list]
engineered_features = pd.concat([df[binary_list],continuous_features],axis=1)
if train==True:
# binarize sucess
# combine title and text
df['requester_received_pizza'] = np.where(df['requester_received_pizza'] == True, 1, 0)
# Get vectorized fields
vec = TfidfVectorizer(stop_words='english',sublinear_tf=1, max_features=max_feature_length)
text_features = df['request_text_n_title'].values.copy()
vectorized_matrix = vec.fit_transform(text_features)
target = df['requester_received_pizza']
return vec, vectorized_matrix, engineered_features, target
else:
text_features = df['request_text_n_title'].values.copy()
request_id = df['request_id']
return request_id,text_features, engineered_features
In [29]:
# process train.json get vectorizer, train_tfid matrix, engineered features, and target
vec, train_tfid_matrix, train_engr_features,train_target = process('train.json',best_max_feature)
# Model 2 on combined tfid_matrix and engr features
combined_features = np.append(train_engr_features.values, train_tfid_matrix.toarray(), axis = 1)
lr2 = LogisticRegression(n_jobs=-1,class_weight='balanced',penalty = 'l2').fit(combined_features, train_target)
# get test
request_id,test_text_features, test_engr_features = process('test.json',best_max_feature,train=False)
test_tfid_matrix = vec.transform(test_text_features)
combined_test_features = np.append(test_engr_features.values,test_tfid_matrix.toarray(),axis=1)
# predict
predict2 = lr2.predict(combined_test_features)[:,np.newaxis]
# print type(predictions_test)
sub_2 = np.append(request_id.values[:, np.newaxis], predict2, axis = 1)
sub_2_df = pd.DataFrame(data=sub_2, columns=['request_id', 'requester_received_pizza']) # 1st row as the column names
sub_2_df.to_csv("submission_2.csv", sep=',', header=True, mode='w', index=0)
"""We achieved AUC of 0.60278 with this submission"""
print "Kaggle submission 2, result AUC = 0.60278"