In [1]:
import numpy as np
from sklearn.externals import joblib
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn import cluster
from scipy import stats
import pickle
import pandas as pd
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import offsetbox
from sklearn import (manifold, datasets, decomposition, ensemble,
discriminant_analysis, random_projection)
import numpy as np # a conventional alias
from sklearn.feature_extraction.text import CountVectorizer
from matplotlib.pyplot import *
from sklearn.decomposition import TruncatedSVD
from sklearn import preprocessing
from gensim import models
import gensim
from gensim import corpora
from gensim.models import TfidfModel
from gensim.models import LsiModel
from gensim.similarities import MatrixSimilarity
from gensim.models import Word2Vec
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn.cluster import KMeans
import logging
# logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
from IPython.display import display
from time import time
import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')
%matplotlib notebook
In [2]:
df = pd.read_csv('data/windowbin/csv/all_dynamic_topic.csv')
# display(df)
index = df.iloc[:,2:3].values
# display(index)
df.drop(df.columns[[0,1,]], axis=1, inplace=True)
display(df.head())
display(df.iloc[:,1:].head())# used for corpus
# df1 = df.apply(lambda row: row.astype(str).str.contains('iran').any(), axis=1)
# display(df[df.apply(lambda row: row.astype(str).str.contains('iran').any(), axis=1)])
In [3]:
def load_w2v(word2vec_model_file):
# load the finished model from disk
word2vec_model = Word2Vec.load(word2vec_model_file)
word2vec_model.init_sims(replace=True)
return word2vec_model
In [4]:
word2vec_model_file = '/home/sonic/sonic/eosdb/data/eos/word2vec_model_all.model'
word2vec_model = load_w2v(word2vec_model_file)
In [5]:
def document_vector(word2vec_model, doc):
# remove out-of-vocabulary words
doc = [word for word in doc if word in word2vec_model.wv.vocab]
return np.mean(word2vec_model[doc], axis=0)
In [6]:
def document_to_vector(word2vec_model, doc):
# remove out-of-vocabulary words
doc = [word for word in doc if word in word2vec_model.wv.vocab]
return np.array(word2vec_model[doc])
In [7]:
def kmeans(X, figName='', metric='euclidean'):
estimators = {
'4': KMeans(n_clusters=4),
'6': KMeans(n_clusters=6),
'10': KMeans(n_clusters=10),
'15': KMeans(n_clusters=15),
'20': KMeans(n_clusters=20),
'30': KMeans(n_clusters=30),
'40': KMeans(n_clusters=40),
'50': KMeans(n_clusters=50),
'70': KMeans(n_clusters=70),
'80': KMeans(n_clusters=80),
'100': KMeans(n_clusters=100),
'150': KMeans(n_clusters=150),
'200': KMeans(n_clusters=200),
}
fignum = 1
minsil=-1
bestlabels=[]
for name, est in estimators.items():
fig = plt.figure(fignum, figsize=(8, 8))
plt.clf()
ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)
plt.cla()
est.fit(X)
labels = est.labels_
centroids = est.cluster_centers_
sil=silhouette_score(X, labels, metric=metric)
print ("silhouette of " + name+"=" +str(sil))
if sil>minsil:
bestlabels=labels
minsil=sil
numOfCluster=name
colormap = plt.cm.gist_ncar # nipy_spectral, Set1,Paired
colorst = [colormap(i) for i in np.linspace(0, 0.9, int(name))]
colors=[]
for i in labels:
colors.append(colorst[i])
ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=colors) #X[:,3] means all rows column=3 ''''''c=labels.astype(np.float)
ax.w_xaxis.set_ticklabels([])
ax.w_yaxis.set_ticklabels([])
ax.w_zaxis.set_ticklabels([])
ax.set_xlabel(name)
fignum = fignum + 1
plt.title("%s k:%s" % (figName, name))
plt.show()
print ("Best silhouette =" + str(minsil))
return numOfCluster,bestlabels
In [8]:
def analyze(X, figName='', metric='euclidean', show=False):
# range_n_clusters = [2, 3, 4, 5, 6, 8, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 120, 140, 160, 180, 200]
range_n_clusters = [30, 40, 50, 60, 70]
best_k = 0
min_score = 0
bestlabels=[]
for n_clusters in range_n_clusters:
# Create a subplot with 1 row and 2 columns
fig, (ax1, ax2) = plt.subplots(1, 2)
fig.set_size_inches(12, 7)
# The 1st subplot is the silhouette plot
# The silhouette coefficient can range from -1, 1 but in this example all
# lie within [-0.1, 1]
ax1.set_xlim([-0.1, 1])
# The (n_clusters+1)*10 is for inserting blank space between silhouette
# plots of individual clusters, to demarcate them clearly.
ax1.set_ylim([0, len(X) + (n_clusters + 1) * 10])
# Initialize the clusterer with n_clusters value and a random generator
# seed of 10 for reproducibility.
clusterer = KMeans(n_clusters=n_clusters, random_state=42)
cluster_labels = clusterer.fit_predict(X)
# The silhouette_score gives the average value for all the samples.
# This gives a perspective into the density and separation of the formed
# clusters
silhouette_avg = silhouette_score(X, cluster_labels, metric=metric)
print("For n_clusters =", n_clusters,
"The average silhouette_score is :", silhouette_avg)
if silhouette_avg > min_score:
bestlabels = cluster_labels
min_score=silhouette_avg
best_k=n_clusters
# Compute the silhouette scores for each sample
sample_silhouette_values = silhouette_samples(X, cluster_labels)
y_lower = 10
for i in range(n_clusters):
# Aggregate the silhouette scores for samples belonging to
# cluster i, and sort them
ith_cluster_silhouette_values = \
sample_silhouette_values[cluster_labels == i]
ith_cluster_silhouette_values.sort()
size_cluster_i = ith_cluster_silhouette_values.shape[0]
y_upper = y_lower + size_cluster_i
color = cm.spectral(float(i) / n_clusters)
ax1.fill_betweenx(np.arange(y_lower, y_upper),
0, ith_cluster_silhouette_values,
facecolor=color, edgecolor=color, alpha=0.7)
# Label the silhouette plots with their cluster numbers at the middle
ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))
# Compute the new y_lower for next plot
y_lower = y_upper + 10 # 10 for the 0 samples
ax1.set_title("clusters silhouette plot")
ax1.set_xlabel("The silhouette coefficient values")
ax1.set_ylabel("Cluster label")
# The vertical line for average silhouette score of all the values
ax1.axvline(x=silhouette_avg, color="red", linestyle="--")
ax1.set_yticks([]) # Clear the yaxis labels / ticks
ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])
# 2nd Plot showing the actual clusters formed
# plt.cm.gist_ncar or nipy_spectral
colors = cm.spectral(cluster_labels.astype(float) / n_clusters)
ax2.scatter(X[:, 0], X[:, 1], X[:, 2], c=colors)
ax2.set_xlabel(n_clusters)
# Labeling the clusters
centers = clusterer.cluster_centers_
# Draw white circles at cluster centers
ax2.scatter(centers[:, 0], centers[:, 1], marker='o',
c="white", alpha=1, s=200, edgecolor='k')
for i, c in enumerate(centers):
ax2.scatter(c[0], c[1], marker='$%d$' % i, alpha=1, s=50, edgecolor='k')
ax2.set_title("clustere %s k:%s" % (figName, n_clusters))
# ax2.set_xlabel("Feature space for the 1st feature")
# ax2.set_ylabel("Feature space for the 2nd feature")
plt.suptitle(("Silhouette analysis for KMeans clustering on topic space "
"with n_clusters = %d" % n_clusters), fontsize=12, fontweight='bold')
# Interactive figure
if (show):
fig2 = plt.figure(figsize=(8, 8))
ax3 = Axes3D(fig2, rect=[0, 0, .95, 1], elev=48, azim=134)
ax3.scatter(X[:, 0], X[:, 1], X[:, 2], c=colors)
ax3.set_xlabel(n_clusters)
ax3.w_xaxis.set_ticklabels([])
ax3.w_yaxis.set_ticklabels([])
ax3.w_zaxis.set_ticklabels([])
ax3.set_title("%s cluster k:%s" % (figName, n_clusters))
plt.show()
return best_k, bestlabels
In [9]:
def getCorpus():
corpus = df.iloc[:,3:].values.tolist()
return corpus
In [10]:
%%time
corpus = getCorpus()
topic_w2v = np.array([document_vector(word2vec_model, doc) for doc in corpus])
X_embedded = TSNE(n_components=3, init='pca', verbose=2).fit_transform(topic_w2v)
# print(topic_w2v[0])
# n_components =20
# print("number of components: {}".format(n_components))
# X_reduced = TruncatedSVD(n_components=n_components, random_state=42).fit_transform(topic_w2v)
# print(X_reduced.shape)
In [11]:
%%time
# No scaling
# numofClusters, bestlabels = kmeans(X_embedded)
best_k, bestlabels = analyze(X_embedded, figName="W2V")
print('best number of clusters: %s' % best_k)
df['label_w2v_no'] = bestlabels
display(df.head())
In [12]:
%%time
# Scaled
X_embedded_scaled = preprocessing.scale(X_embedded)
best_k, bestlabels = analyze(X_embedded_scaled, figName="W2V scale")
print('best number of clusters: %s' % best_k)
df['label_w2v_scale'] = bestlabels
display(df.head())
In [13]:
%%time
# Normalized
X_normalized = preprocessing.normalize(X_embedded, norm='l2')
best_k, bestlabels = analyze(X_normalized, figName="W2V normalize")
print('best number of clusters: %s' % best_k)
df['label_w2v_normalize'] = bestlabels
display(df.head())
In [14]:
X_wmd_distance_eos = pd.read_pickle('data/df_X_wmd_distance_eos.plk')
# print(X_wmd_distance_eos)
X_wmd_distance_eos = TSNE(n_components=3, init='pca', verbose=2).fit_transform(X_wmd_distance_eos)
In [15]:
# Pairewise distance
best_k, bestlabels = analyze(X_wmd_distance_eos, figName="WMD distance")
print('best number of clusters: %s' % best_k)
df['label_wmd_distance'] = bestlabels
display(df.head())
In [16]:
display(df.loc[df['label_wmd_distance'] == 5].head())
In [17]:
# Normalized
X_wmd_normalized = preprocessing.normalize(X_wmd_distance_eos, norm='l2')
pca = PCA(n_components=3)
pca.fit(X_wmd_normalized)
X_wmd_normalized = pca.transform(X_wmd_normalized)
best_k, bestlabels = analyze(X_wmd_normalized, figName="WMD normalized")
print('best number of clusters: %s' % best_k)
df['label_wmd_normalize'] = bestlabels
In [18]:
# Scaled
X_wmd_scaled = preprocessing.scale(X_wmd_distance_eos)
pca = PCA(n_components=3)
pca.fit(X_wmd_scaled)
X_wmd_scaled = pca.transform(X_wmd_scaled)
best_k, bestlabels = analyze(X_wmd_scaled, figName="WMD scaled")
df['label_wmd_scale'] = best_k
display(df.head())
In [19]:
%%time
corpus_all = []
for corpus_line in getCorpus():
corpus_all.append(u' '.join(str(e) for e in corpus_line))
print(corpus_all[0])
vectorizer = CountVectorizer(max_df=0.5, min_df=5)
# vectorizer = CountVectorizer(max_df=0.5)
X_tfidf = vectorizer.fit_transform(corpus_all) # a sparse matrix
vocab = vectorizer.get_feature_names() # a list
print(len(vocab))
print(X_tfidf.shape)
In [20]:
%%time
# Normalized
X_tfidf_normalized = preprocessing.normalize(X_tfidf, norm='l2')
# LSA
X_tfidf_normalized = decomposition.TruncatedSVD(n_components=50).fit_transform(X_tfidf_normalized)
X_tfidf_normalized = TSNE(n_components=3, init='pca', verbose=2).fit_transform(X_tfidf_normalized)
best_k, bestlabels = analyze(X_tfidf_normalized, figName="TF-IDF normalized")
print('best number of clusters: %s' % best_k)
df['label_tfidf_normalize'] = bestlabels
display(df.head())
In [21]:
display(df.loc[df['label_tfidf_normalize'] == 7])
In [22]:
df.to_csv('data/windowbin/csv/result_all_dynamic_topic.csv')
In [ ]:
In [ ]: