In [16]:
# -*- coding:utf-8 -*-
%matplotlib inline
import sys, os
root_dir = os.path.join(os.path.dirname(os.path.realpath('__file__')), '..')
sys.path.append(root_dir)
import dmr
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict
def init_lda(datfilepath, K=5, alpha=0.1, beta=0.01):
corpus = dmr.Corpus.read(datfilepath)
voca = dmr.Vocabulary()
docs = voca.read_corpus(corpus)
lda = dmr.LDA(K, alpha, beta, docs, voca.size())
return corpus, voca, docs, lda
datfilepath = os.path.join(root_dir, 'dat', 'LDA.dat')
# learning
corpus, voca, docs, lda = init_lda(datfilepath)
lda.learning(iteration=100, voca=voca)
# word probability of each topic
wdist = lda.word_dist_with_voca(voca)
for k in wdist:
print("TOPIC", k)
print("\t".join([w for w in wdist[k]]))
print("\t".join(["%0.2f" % wdist[k][w] for w in wdist[k]]))
print()
# topic probability of each document
tdist = lda.topicdist()
for first_letter in ["a", "b", "c", "d", "e"]:
for doc, td in zip(corpus, tdist):
if doc[0].startswith(first_letter):
print("DOC", "Words: ", doc, "Max topic: ", np.argmax(td), "Max prob.: ", np.max(td))
TOPIC 0
di dd dg dc de da db dh df dj
0.08 0.14 0.10 0.09 0.10 0.08 0.10 0.10 0.09 0.11
TOPIC 1
ci ch cb ce ca cj cc cg cf cd
0.07 0.11 0.07 0.11 0.13 0.07 0.10 0.11 0.11 0.12
TOPIC 2
bg bc bi bh be bj bb ba bf bd
0.09 0.15 0.07 0.07 0.13 0.06 0.13 0.12 0.08 0.10
TOPIC 3
ac ab aj af ad ah ae ai ag aa
0.08 0.08 0.08 0.11 0.11 0.08 0.09 0.17 0.09 0.09
TOPIC 4
ed ei eh ee ej ec ef eb eg ea
0.08 0.07 0.13 0.09 0.13 0.08 0.12 0.12 0.07 0.11
DOC Words: ['ai', 'ab', 'ag', 'ah', 'ah', 'ai', 'ab', 'af', 'aj', 'ai'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ai', 'ai', 'ac', 'ad', 'ac', 'aa', 'ai', 'ai', 'ad', 'ai'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ai', 'af', 'aj', 'aa', 'aj', 'ag', 'af', 'ad', 'ab', 'ai'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ae', 'aj', 'ag', 'af', 'ah', 'ai', 'ai', 'aj', 'ac', 'ai'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ai', 'ab', 'ab', 'aa', 'ad', 'ai', 'ai', 'ae', 'ae', 'aa'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ae', 'aj', 'aa', 'ag', 'ae', 'ac', 'ae', 'ag', 'ad', 'ad'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['af', 'aa', 'ai', 'af', 'ae', 'ah', 'ae', 'ab', 'ad', 'ad'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ai', 'ah', 'aa', 'ai', 'ag', 'ai', 'aj', 'ai', 'ad', 'ag'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ad', 'ai', 'ae', 'ab', 'ae', 'af', 'aa', 'ad', 'ab', 'ae'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ah', 'ag', 'af', 'aj', 'ag', 'af', 'ac', 'ah', 'ab', 'aj'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['aj', 'ah', 'af', 'ab', 'af', 'ag', 'ag', 'ai', 'ah', 'af'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ad', 'af', 'ai', 'ae', 'ab', 'ah', 'ai', 'ab', 'ac', 'ab'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ab', 'ad', 'aj', 'ac', 'ai', 'ac', 'ae', 'ai', 'ac', 'aj'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ah', 'ai', 'ac', 'ad', 'ad', 'ag', 'aa', 'ad', 'ag', 'ad'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ai', 'af', 'af', 'af', 'ab', 'aa', 'aa', 'aa', 'aa', 'aa'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ac', 'ah', 'ah', 'ac', 'ag', 'ab', 'ad', 'ah', 'aa', 'af'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ag', 'ai', 'ai', 'ag', 'ad', 'ae', 'ae', 'ad', 'ag', 'af'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ac', 'ac', 'aj', 'ag', 'ah', 'ai', 'aj', 'ai', 'ac', 'ae'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['ae', 'aj', 'ae', 'aa', 'ac', 'af', 'af', 'ad', 'aa', 'aa'] Max topic: 3 Max prob.: 0.961904761905
DOC Words: ['be', 'bd', 'ba', 'bd', 'bf', 'ba', 'bc', 'bd', 'bi', 'bb'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bd', 'ba', 'bf', 'ba', 'bb', 'bc', 'be', 'bc', 'ba', 'bd'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bj', 'bj', 'bh', 'bd', 'bc', 'bd', 'bj', 'bh', 'bh', 'bf'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bc', 'bc', 'bi', 'bb', 'bf', 'bi', 'be', 'ba', 'bc', 'bf'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bd', 'bh', 'bd', 'bc', 'bb', 'bb', 'bc', 'bb', 'be', 'bc'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['ba', 'bg', 'ba', 'be', 'bi', 'be', 'bd', 'bd', 'bi', 'bi'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bc', 'bf', 'bc', 'bd', 'bf', 'bh', 'bc', 'bh', 'bb', 'bg'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bh', 'be', 'bj', 'bc', 'ba', 'bi', 'bc', 'bh', 'bi', 'be'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bc', 'bb', 'bd', 'be', 'bb', 'bb', 'ba', 'bh', 'bi', 'be'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bh', 'bf', 'ba', 'be', 'bb', 'bb', 'bg', 'bg', 'ba', 'bc'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bc', 'be', 'bj', 'bj', 'bf', 'be', 'be', 'bj', 'bi', 'bb'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bg', 'bd', 'bc', 'bc', 'bc', 'bg', 'be', 'bg', 'ba', 'be'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bc', 'bb', 'bg', 'bb', 'bh', 'bf', 'bg', 'bb', 'bg', 'bg'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['be', 'be', 'bf', 'bi', 'bi', 'ba', 'bb', 'bf', 'bj', 'bj'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bb', 'ba', 'ba', 'bf', 'bg', 'bd', 'bg', 'bb', 'bh', 'bh'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['be', 'bf', 'bh', 'be', 'be', 'bc', 'bc', 'bb', 'bd', 'ba'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['be', 'bj', 'ba', 'bg', 'bf', 'bd', 'ba', 'ba', 'bb', 'bc'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['ba', 'bc', 'bb', 'be', 'bg', 'ba', 'bg', 'bg', 'bc', 'bi'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['bj', 'bj', 'bc', 'bb', 'ba', 'bb', 'bg', 'bd', 'be', 'bd'] Max topic: 2 Max prob.: 0.961904761905
DOC Words: ['ca', 'ch', 'cf', 'cj', 'ca', 'cc', 'ch', 'cc', 'cj', 'cc'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cc', 'ci', 'ce', 'cd', 'ca', 'ce', 'cd', 'cg', 'cj', 'ci'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cf', 'ca', 'ca', 'cd', 'cb', 'cj', 'cj', 'cg', 'cg', 'ch'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cd', 'cj', 'ch', 'cg', 'cg', 'cc', 'cd', 'cg', 'ca', 'ci'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cc', 'cf', 'cg', 'ce', 'cc', 'cc', 'cb', 'ca', 'cj', 'ca'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['ci', 'cd', 'ca', 'ci', 'ci', 'cb', 'ca', 'cf', 'ci', 'cc'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['ce', 'cb', 'ch', 'ce', 'ca', 'cc', 'ch', 'ce', 'ca', 'cc'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['ch', 'cg', 'cd', 'cf', 'ce', 'cd', 'ce', 'cf', 'cb', 'cd'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cj', 'ci', 'cb', 'cd', 'ci', 'cb', 'cf', 'ci', 'ca', 'cd'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['ca', 'ch', 'cg', 'ca', 'ce', 'cc', 'ch', 'cf', 'cc', 'ci'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cf', 'cg', 'cg', 'ce', 'cb', 'cf', 'ce', 'cf', 'cf', 'cg'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cd', 'cc', 'cf', 'cj', 'ch', 'cd', 'ce', 'ch', 'cg', 'ch'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cd', 'cc', 'ca', 'ch', 'ce', 'cg', 'ca', 'cf', 'cd', 'cd'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['cf', 'ca', 'cd', 'cj', 'cj', 'ch', 'ca', 'ce', 'cb', 'cd'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['ce', 'ce', 'cg', 'ca', 'cb', 'ch', 'cg', 'cf', 'cf', 'ch'] Max topic: 1 Max prob.: 0.961904761905
DOC Words: ['dd', 'dd', 'dh', 'da', 'db', 'dj', 'dj', 'da', 'de', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dc', 'dh', 'dc', 'da', 'da', 'de', 'df', 'df', 'dg', 'di'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dc', 'da', 'dd', 'df', 'dj', 'de', 'de', 'dg', 'de', 'de'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['de', 'de', 'di', 'de', 'dd', 'dh', 'df', 'df', 'da', 'db'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dd', 'dc', 'dd', 'de', 'db', 'dc', 'dj', 'db', 'de', 'dg'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['di', 'dc', 'dd', 'da', 'da', 'dg', 'da', 'dg', 'dd', 'dd'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dg', 'dj', 'db', 'dg', 'di', 'di', 'dd', 'dc', 'dd', 'dg'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dg', 'df', 'dh', 'da', 'di', 'de', 'dg', 'df', 'di', 'dc'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dd', 'dj', 'dh', 'df', 'dd', 'de', 'df', 'dd', 'dd', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dh', 'da', 'dd', 'di', 'dh', 'dh', 'db', 'di', 'de', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['df', 'dh', 'dd', 'dg', 'dg', 'dh', 'dj', 'db', 'dj', 'dg'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['df', 'dg', 'dd', 'dc', 'dj', 'di', 'db', 'de', 'da', 'di'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dj', 'df', 'df', 'db', 'dh', 'di', 'dg', 'de', 'dh', 'dd'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dd', 'dg', 'de', 'dh', 'dd', 'da', 'df', 'dj', 'dd', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['da', 'dd', 'dg', 'db', 'dj', 'dc', 'dj', 'de', 'dj', 'db'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dc', 'de', 'dj', 'dh', 'de', 'dj', 'de', 'db', 'dc', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['df', 'dd', 'di', 'dg', 'de', 'dg', 'dd', 'dg', 'dc', 'dg'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['df', 'dj', 'de', 'dg', 'df', 'db', 'dd', 'dd', 'di', 'dj'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dd', 'dc', 'dj', 'dj', 'dd', 'dc', 'df', 'de', 'db', 'df'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dh', 'dj', 'dc', 'de', 'dj', 'da', 'dg', 'dj', 'dc', 'de'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['da', 'df', 'de', 'da', 'dc', 'dd', 'db', 'dh', 'db', 'dd'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dc', 'df', 'dc', 'dc', 'dd', 'dc', 'dj', 'dd', 'di', 'df'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['di', 'db', 'de', 'dc', 'dj', 'dh', 'de', 'db', 'di', 'dc'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dd', 'de', 'db', 'dg', 'dc', 'df', 'de', 'dj', 'db', 'df'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['db', 'dj', 'db', 'dg', 'dj', 'di', 'dg', 'dd', 'da', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dg', 'dj', 'dd', 'dh', 'db', 'dd', 'dh', 'di', 'di', 'de'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dg', 'dj', 'dg', 'dc', 'dj', 'dd', 'df', 'dh', 'dj', 'da'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['df', 'dd', 'db', 'dh', 'dd', 'di', 'df', 'df', 'db', 'dj'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dc', 'db', 'da', 'db', 'da', 'db', 'db', 'dg', 'di', 'di'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dc', 'dh', 'dd', 'de', 'dj', 'dh', 'da', 'da', 'db', 'di'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['df', 'da', 'db', 'db', 'dd', 'dh', 'dj', 'dd', 'dh', 'da'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['dd', 'dg', 'di', 'di', 'dc', 'db', 'da', 'di', 'dh', 'dh'] Max topic: 0 Max prob.: 0.961904761905
DOC Words: ['ef', 'ea', 'ed', 'ed', 'eh', 'ej', 'ed', 'ef', 'ec', 'ee'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['eb', 'ee', 'ej', 'ei', 'eb', 'eb', 'eh', 'ej', 'ej', 'ed'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ef', 'ec', 'ef', 'eh', 'eh', 'eg', 'eb', 'eg', 'eh', 'ec'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ed', 'eb', 'ej', 'ef', 'ej', 'ej', 'ec', 'ea', 'ej', 'eb'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['eb', 'eh', 'eg', 'ej', 'ee', 'eb', 'ef', 'ej', 'eh', 'eb'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ee', 'ea', 'ea', 'ei', 'ee', 'eg', 'ej', 'ed', 'ed', 'ec'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ei', 'ea', 'ei', 'ed', 'eg', 'ej', 'ed', 'ec', 'eh', 'ea'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ef', 'eb', 'ef', 'ei', 'ec', 'ej', 'ej', 'ef', 'eg', 'ee'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['eg', 'eh', 'ee', 'ea', 'ea', 'eh', 'ed', 'ed', 'ea', 'ea'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['eg', 'eb', 'ee', 'ea', 'ee', 'eh', 'eh', 'eh', 'eg', 'ec'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ef', 'ef', 'ed', 'ei', 'ea', 'eb', 'ef', 'ee', 'eb', 'ef'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['eb', 'eh', 'ei', 'ea', 'eh', 'ef', 'ej', 'ej', 'eg', 'ei'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['eh', 'eb', 'ec', 'ef', 'ea', 'ef', 'ei', 'ej', 'ei', 'ec'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ef', 'eh', 'ec', 'ee', 'eb', 'eb', 'eb', 'eh', 'ej', 'ea'] Max topic: 4 Max prob.: 0.961904761905
DOC Words: ['ea', 'ee', 'ei', 'ea', 'ee', 'ef', 'eh', 'ec', 'ee', 'ej'] Max topic: 4 Max prob.: 0.961904761905
In [ ]:
Content source: mpkato/dmr
Similar notebooks: