In [1]:
import sys, os
import nltk
import pandas as pd
In [2]:
from collections import defaultdict
from gensim import corpora, models, similarities
def make_dictionary(documents):
"""
construct a dictionary, i.e. mapping btwn word ids and their freq of occurence in the whole corpus
filter dictionary to remove stopwords and words occuring < min_count times
input: documents is an iterable consisting of all the words in the corpus
output: filtered dictionary
"""
dictionary = corpora.Dictionary(documents)
stop_words = nltk.corpus.stopwords.words('english')
min_count = 2
stop_ids = [dictionary.token2id[word] for word in stop_words
if word in dictionary.token2id]
rare_ids = [id for id, freq in dictionary.dfs.items()
if freq < min_count]
dictionary.filter_tokens(stop_ids + rare_ids)
dictionary.compactify()
return(dictionary)
def make_corpus(documents):
"""
"""
dictionary = make_dictionary(documents)
# convert corpus to vectors using bag-of-words representation, i.e. tuples of word indices and word counts
corpus = [dictionary.doc2bow(words) for words in documents]
return(corpus, dictionary)
def make_lsi_similarity_matrix(tfidf_corpus, dictionary):
"""
construct LSI (latent semantic indexing) model on Tfidf-transformed corpus, print model topics,
return similarity matrix.
"""
# construct model
lsi = models.lsimodel.LsiModel(tfidf_corpus, id2word=dictionary, num_topics=200)
lsi.save('lsi-model.save')
# create similarity matrix
matsim = similarities.MatrixSimilarity(lsi[tfidf_corpus], num_best=4)
return(matsim)
def make_lda_similarity_matrix(corpus, dictionary):
"""
Latent Dirichlet Allocation (LDA) model
"""
# construct model
lda = models.ldamodel.LdaModel(corpus, id2word=dictionary, num_topics=200)
lda.save('lda-model.save')
# create similarity matrix
matsim = similarities.MatrixSimilarity(lda[corpus], num_best=4)
return(matsim)
In [3]:
# Read database of data
input_fname="AutismParentMagazine-posts.csv"
output_fname=input_fname.replace(".csv","-tokens.csv")
os.chdir('../data')
df=pd.read_csv(input_fname,index_col=0)
df.head(5)
Out[3]:
In [4]:
# Tokenize data
import nltk
tokenizer = nltk.RegexpTokenizer(r'\w+')
# Get list of tokens from text in first article:
text = df['text'][0].lower()
ttext = tokenizer.tokenize(text)
print( text )
print( ttext )
In [5]:
# Get a column with list of tokens:
# 1) convert to lower case
# 2) get tokens
# 2) save data in a new column (tokens)
df['tokens'] = df['text'].map(lambda x: tokenizer.tokenize(x.lower()))
df['tokens'].head(5)
Out[5]:
In [8]:
nltk.download()
In [ ]:
# Save dataframe with tokens into files
df.to_csv(output_fname)
In [6]:
# Get similarity matrices
documents = df['tokens'].values
corpus,dictionary = make_corpus(documents)
#Save corpus into file
import pickle
pickle.dump(dictionary,open("dictionary.save","wb"))
pickle.dump(corpus,open("corpus.save", "wb"))
tfidf = models.TfidfModel(corpus)
tfidf.save('tfidf.save')
lsi_matsim = make_lsi_similarity_matrix(tfidf[corpus], dictionary)
lda_matsim = make_lda_similarity_matrix(corpus, dictionary)
# The models are saved into files in the above routines
# Save similarity matrices too:
pickle.dump(lsi_matsim,open("lsi-matsim.save","wb"))
pickle.dump(lda_matsim,open("lda-matsim.save","wb"))
In [ ]: