In [1]:
from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(min_df=1)
# given some content...
content = ["How to format my hard disk", " Hard disk format problems "]
X = vectorizer.fit_transform(content)
feature_names = vectorizer.get_feature_names()
print("Feature names: {}".format(feature_names))
print(X.toarray().transpose())
Description of above data: Each column gives us a boolean (1 or 0) value letting us know if each word appears in the sentence (from content). Sentence 1 (content[0]) contains all words but "problems".
In [17]:
posts = [
"This is a toy post about machine learning. Actually, it contains not much interesting stuff.",
"Imaging databases provide storage capabilities.",
"Most imaging databases save images permanently.",
"Imaging databases store data.",
"Imaging databases store data. Imaging databases store data. Imaging databases store data.",
]
# Create a training set
vectorizer = CountVectorizer(min_df=1)
X_train = vectorizer.fit_transform(posts)
num_samples, num_features = X_train.shape
print("#samples: {}, #features: {}".format(num_samples, num_features))
print(vectorizer.get_feature_names())
# create a new post
new_post = "imaging databases"
new_post_vec = vectorizer.transform([new_post])
# a naive similarity measure (which uses the full ndarray of the new post)
import scipy as sp
def dist_raw(v1, v2):
delta = v1 - v2
# norm: Euclidean norm (shortest distance)
return sp.linalg.norm(delta.toarray())
# Find distances among all posts
import sys
def find_distances(vectorizer, new_post, posts, dist_func=dist_raw):
X_train = vectorizer.fit_transform(posts)
new_post_vec = vectorizer.transform([new_post])
num_samples, num_features = X_train.shape
print("----------------------------------------")
print("#samples: {}, #features: {}".format(num_samples, num_features))
print(vectorizer.get_feature_names())
print("----------------------------------------")
best_dist = sys.maxsize
best_i = None
for i in range(0, num_samples):
post = posts[i]
if post == new_post:
continue
post_vec = X_train.getrow(i)
d = dist_func(post_vec, new_post_vec)
print("- Post %i with dist=%.2f: %s" % (i, d, post))
if d < best_dist:
best_dist = d
best_i = i
print("Best post is %i with dist=%.2f" % (best_i, best_dist))
find_distances(vectorizer, new_post, posts)
# explore the vectors for posts 3 & 4 since they all contain the same words
print("\nVectors for what should be similar sentences:")
print(X_train.getrow(3).toarray())
print(X_train.getrow(4).toarray())
In [18]:
# Normalize the vectors, and try again
def dist_norm(v1, v2):
v1_normalized = v1 / sp.linalg.norm(v1.toarray())
v2_normalized = v2 / sp.linalg.norm(v2.toarray())
delta = v1_normalized - v2_normalized
return sp.linalg.norm(delta.toarray())
find_distances(vectorizer, new_post, posts, dist_func=dist_norm)
In [19]:
# using stop words; i.e. removing "noise" / useless info
# use common english stop words (can also provide a list of specific words)
vectorizer = CountVectorizer(min_df=1, stop_words='english')
print("Some of our stop words: {}".format(", ".join(sorted(vectorizer.get_stop_words())[0:20])))
# construct a new training set
X_train = vectorizer.fit_transform(posts)
num_samples, num_features = X_train.shape
print("#samples: {}, #features: {}".format(num_samples, num_features))
print(vectorizer.get_feature_names())
In [20]:
# Using NLTK for stemming (reducing words to their specific word stem)
import nltk.stem as ns
s = ns.SnowballStemmer('english')
print(s.stem("graphics"))
print(s.stem("imaging"))
print(s.stem("image"))
print(s.stem("imagination"))
print(s.stem("imagine"))
In [21]:
# stem our posts before verctorizing
import nltk.stem
english_stemmer = nltk.stem.SnowballStemmer('english')
class StemmedCountVectorizer(CountVectorizer):
def build_analyzer(self):
analyzer = super().build_analyzer()
return lambda doc: (english_stemmer.stem(w) for w in analyzer(doc))
vectorizer = StemmedCountVectorizer(min_df=1, stop_words='english')
find_distances(vectorizer, new_post, posts, dist_func=dist_norm)
In [36]:
# Finally cluster some stuff
# Using data sets from: http://mlcomp.org/datasets/379
import sklearn.datasets
all_data = sklearn.datasets.fetch_20newsgroups(subset='all')
#print("Files: {}".format(len(all_data.filenames)))
#print("Target Names: {}".format(", ".join(all_data.target_names)))
# set up some training data
groups = [
'comp.graphics',
'comp.os.ms-windows.misc',
'comp.sys.ibm.pc.hardware',
'comp.sys.mac.hardware',
'comp.windows.x',
'sci.space',
]
train_data = sklearn.datasets.fetch_20newsgroups(subset='train', categories=groups)
test_data = sklearn.datasets.fetch_20newsgroups(subset='test', categories=groups)
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk.stem
english_stemmer = nltk.stem.SnowballStemmer('english')
class StemmedTfidfVectorizer(TfidfVectorizer):
def build_analyzer(self):
analyzer = super().build_analyzer()
return lambda doc: (english_stemmer.stem(w) for w in analyzer(doc))
vectorizer = StemmedTfidfVectorizer(min_df=10, max_df=0.5, stop_words='english', decode_error='ignore')
vectorized = vectorizer.fit_transform(train_data.data)
num_samples, num_features = vectorized.shape
print("#samples: {}\n#features: {}".format(num_samples, num_features))
# NOW, do the clustering
from sklearn.cluster import KMeans
km = KMeans(n_clusters=50, init='random', n_init=1, verbose=1, random_state=3)
km.fit(vectorized)
print(km.labels_)
print(km.labels_.shape)
In [50]:
# Test out on a new mailling list post.
new_post = """Disk drive problems. Hi, I have a problem with my hard disk.
After 1 year it is working only sporadically now.
I tried to format it, but now it doesn't boot any more.
Any ideas? Thanks.
"""
# vectorize the new post, and predict it's grouping
new_post_vec = vectorizer.transform([new_post])
new_post_label = km.predict(new_post_vec)[0]
print("New post label: {}".format(new_post_label))
# find similar...
similar_indices = (km.labels_ == new_post_label).nonzero()[0]
# build a list of similar posts
similar = []
for i in similar_indices:
dist = sp.linalg.norm((new_post_vec - vectorized[i]).toarray())
similar.append((dist, train_data.data[i]))
similar = sorted(similar)
print("Found {} similar posts.".format(len(similar)))
a = similar[0]
b = similar[int(len(similar) / 10)]
c = similar[int(len(similar) / 2)]
fmt = "\n-- similarity score: {} --\n{}\n--------------------------\n"
print(fmt.format(a[0], a[1]))
print(fmt.format(b[0], b[1]))
print(fmt.format(c[0], c[1]))
In [ ]: