In [2]:
import turicreate as tc
In [6]:
sf = tc.SFrame.read_csv("/Users/datalab/bigdata/cjc/w15",
header=False)
Finished parsing file /Users/datalab/bigdata/cjc/w15
Parsing completed. Parsed 100 lines in 0.447868 secs.
------------------------------------------------------
Inferred types from first 100 line(s) of file as
column_type_hints=[str]
If parsing fails due to incorrect types, you can correct
the inferred type list above and pass it to read_csv in
the column_type_hints argument
------------------------------------------------------
Read 12278 lines. Lines per second: 16914.5
Finished parsing file /Users/datalab/bigdata/cjc/w15
Parsing completed. Parsed 72269 lines in 1.73223 secs.
In [7]:
sf
Out[7]:
X1
aynrand born and educated
in russia rand migrated ...
asphalt in american
english asphalt or ...
actinopterygii the
actinopterygii consti ...
altaiclanguages these
language families share ...
argon the name argon is
derived from the greek ...
augustderleth a 1938
guggenheim fellow der ...
amateur amateurism can be
seen in both a negative ...
assemblyline an assembly
line is a manufacturing ...
astronomicalunit an
astronomical unit ...
abbess an abbess latin
abbatissa feminine form ...
[72269 rows x 1 columns]
Note: Only the head of the SFrame is printed.
You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.
In [8]:
dir(sf['X1'])
Out[8]:
['_SArray__check_min_observations',
'__abs__',
'__add__',
'__and__',
'__bool__',
'__class__',
'__contains__',
'__copy__',
'__deepcopy__',
'__delattr__',
'__dir__',
'__div__',
'__doc__',
'__eq__',
'__floordiv__',
'__format__',
'__ge__',
'__get_content_identifier__',
'__getattribute__',
'__getitem__',
'__gt__',
'__has_size__',
'__hash__',
'__init__',
'__is_materialized__',
'__iter__',
'__le__',
'__len__',
'__lt__',
'__materialize__',
'__mod__',
'__module__',
'__mul__',
'__ne__',
'__neg__',
'__new__',
'__nonzero__',
'__or__',
'__pos__',
'__pow__',
'__proxy__',
'__radd__',
'__rdiv__',
'__reduce__',
'__reduce_ex__',
'__repr__',
'__rfloordiv__',
'__rmod__',
'__rmul__',
'__rpow__',
'__rsub__',
'__rtruediv__',
'__setattr__',
'__sizeof__',
'__slots__',
'__str__',
'__sub__',
'__subclasshook__',
'__truediv__',
'_count_ngrams',
'_count_words',
'_getitem_cache',
'_save_as_text',
'all',
'any',
'append',
'apply',
'argmax',
'argmin',
'astype',
'clip',
'clip_lower',
'clip_upper',
'contains',
'countna',
'cumulative_max',
'cumulative_mean',
'cumulative_min',
'cumulative_std',
'cumulative_sum',
'cumulative_var',
'date_range',
'datetime_to_str',
'dict_has_all_keys',
'dict_has_any_keys',
'dict_keys',
'dict_trim_by_keys',
'dict_trim_by_values',
'dict_values',
'dropna',
'dtype',
'element_slice',
'explore',
'fillna',
'filter',
'filter_by',
'from_const',
'from_sequence',
'hash',
'head',
'is_in',
'is_materialized',
'is_topk',
'item_length',
'materialize',
'max',
'mean',
'min',
'nnz',
'pixel_array_to_image',
'plot',
'random_integers',
'random_split',
'read_json',
'rolling_count',
'rolling_max',
'rolling_mean',
'rolling_min',
'rolling_stdv',
'rolling_sum',
'rolling_var',
'sample',
'save',
'shape',
'show',
'sort',
'split_datetime',
'stack',
'std',
'str_to_datetime',
'sum',
'summary',
'tail',
'to_numpy',
'unique',
'unpack',
'value_counts',
'var',
'vector_slice',
'where']
In [9]:
bow = sf['X1']._count_words()
In [10]:
type(sf['X1'])
Out[10]:
turicreate.data_structures.sarray.SArray
In [11]:
type(bow)
Out[11]:
turicreate.data_structures.sarray.SArray
In [12]:
bow.dict_has_any_keys(['limited'])
Out[12]:
dtype: int
Rows: 72269
[1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, ... ]
In [13]:
bow.dict_values()[0][:20]
Out[13]:
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1]
In [14]:
sf
Out[14]:
X1
aynrand born and educated
in russia rand migrated ...
asphalt in american
english asphalt or ...
actinopterygii the
actinopterygii consti ...
altaiclanguages these
language families share ...
argon the name argon is
derived from the greek ...
augustderleth a 1938
guggenheim fellow der ...
amateur amateurism can be
seen in both a negative ...
assemblyline an assembly
line is a manufacturing ...
astronomicalunit an
astronomical unit ...
abbess an abbess latin
abbatissa feminine form ...
[72269 rows x 1 columns]
Note: Only the head of the SFrame is printed.
You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.
In [15]:
sf['bow'] = bow
In [16]:
sf
Out[16]:
X1
bow
aynrand born and educated
in russia rand migrated ...
{'spoke': 1, '5000': 1,
'follows': 1, 'given' ...
asphalt in american
english asphalt or ...
{'lain': 1, 'commonly':
4, 'has': 6, 'percent': ...
actinopterygii the
actinopterygii consti ...
{'what': 1, 'follows': 1,
'given': 1, ...
altaiclanguages these
language families share ...
{'follows': 2, 'has': 11,
'general': 1, ...
argon the name argon is
derived from the greek ...
{'commonly': 1,
'lattice': 1, ...
augustderleth a 1938
guggenheim fellow der ...
{'rescue': 2, 'amoral':
1, 'lovecraft': 11, ...
amateur amateurism can be
seen in both a negative ...
{'receiving': 1,
'having': 1, 'hand': 1, ...
assemblyline an assembly
line is a manufacturing ...
{'consider': 1, 'world':
1, 'bring': 2, 'pins' ...
astronomicalunit an
astronomical unit ...
{'given': 1, 'ephemeris':
2, 'world': 2, ...
abbess an abbess latin
abbatissa feminine form ...
{'major': 1, 'abbess':
10, 'given': 1, ...
[72269 rows x 2 columns]
Note: Only the head of the SFrame is printed.
You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.
In [17]:
type(sf['bow'])
Out[17]:
turicreate.data_structures.sarray.SArray
In [18]:
len(sf['bow'])
Out[18]:
72269
In [21]:
list(sf['bow'][0].items())[:3]
Out[21]:
[('spoke', 1), ('5000', 1), ('follows', 1)]
In [22]:
sf['tfidf'] = tc.text_analytics.tf_idf(sf['X1'])
In [23]:
sf
Out[23]:
X1
bow
tfidf
aynrand born and educated
in russia rand migrated ...
{'spoke': 1, '5000': 1,
'follows': 1, 'given' ...
{'spoke':
4.830308280673057, ...
asphalt in american
english asphalt or ...
{'lain': 1, 'commonly':
4, 'has': 6, 'percent': ...
{'lain':
8.480100346078947, ...
actinopterygii the
actinopterygii consti ...
{'what': 1, 'follows': 1,
'given': 1, ...
{'what':
2.505781957805935, ...
altaiclanguages these
language families share ...
{'follows': 2, 'has': 11,
'general': 1, ...
{'follows':
7.5113334785280745, ...
argon the name argon is
derived from the greek ...
{'commonly': 1,
'lattice': 1, ...
{'commonly':
3.7717720679882287, ...
augustderleth a 1938
guggenheim fellow der ...
{'rescue': 2, 'amoral':
1, 'lovecraft': 11, ...
{'rescue':
9.19021202607744, ...
amateur amateurism can be
seen in both a negative ...
{'receiving': 1,
'having': 1, 'hand': 1, ...
{'receiving':
4.162612232542636, ...
assemblyline an assembly
line is a manufacturing ...
{'consider': 1, 'world':
1, 'bring': 2, 'pins' ...
{'consider':
4.336965619687414, ...
astronomicalunit an
astronomical unit ...
{'given': 1, 'ephemeris':
2, 'world': 2, ...
{'given':
2.5682202783138064, ...
abbess an abbess latin
abbatissa feminine form ...
{'major': 1, 'abbess':
10, 'given': 1, ...
{'major':
2.356876809458607, ...
[72269 rows x 3 columns]
Note: Only the head of the SFrame is printed.
You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.
In [24]:
list(sf['tfidf'][0].items())[:5]
Out[24]:
[('spoke', 4.830308280673057),
('5000', 4.791220891965009),
('follows', 3.7556667392640373),
('given', 2.5682202783138064),
('percent', 17.481279902908025)]
In [25]:
docs = sf['bow'].dict_trim_by_values(2)
In [27]:
docs = docs.dict_trim_by_keys(
tc.text_analytics.stop_words(),
exclude=True)
In [28]:
help(tc.topic_model.create)
Help on function create in module turicreate.toolkits.topic_model.topic_model:
create(dataset, num_topics=10, initial_topics=None, alpha=None, beta=0.1, num_iterations=10, num_burnin=5, associations=None, verbose=False, print_interval=10, validation_set=None, method='auto')
Create a topic model from the given data set. A topic model assumes each
document is a mixture of a set of topics, where for each topic some words
are more likely than others. One statistical approach to do this is called a
"topic model". This method learns a topic model for the given document
collection.
Parameters
----------
dataset : SArray of type dict or SFrame with a single column of type dict
A bag of words representation of a document corpus.
Each element is a dictionary representing a single document, where
the keys are words and the values are the number of times that word
occurs in that document.
num_topics : int, optional
The number of topics to learn.
initial_topics : SFrame, optional
An SFrame with a column of unique words representing the vocabulary
and a column of dense vectors representing
probability of that word given each topic. When provided,
these values are used to initialize the algorithm.
alpha : float, optional
Hyperparameter that controls the diversity of topics in a document.
Smaller values encourage fewer topics per document.
Provided value must be positive. Default value is 50/num_topics.
beta : float, optional
Hyperparameter that controls the diversity of words in a topic.
Smaller values encourage fewer words per topic. Provided value
must be positive.
num_iterations : int, optional
The number of iterations to perform.
num_burnin : int, optional
The number of iterations to perform when inferring the topics for
documents at prediction time.
verbose : bool, optional
When True, print most probable words for each topic while printing
progress.
print_interval : int, optional
The number of iterations to wait between progress reports.
associations : SFrame, optional
An SFrame with two columns named "word" and "topic" containing words
and the topic id that the word should be associated with. These words
are not considered during learning.
validation_set : SArray of type dict or SFrame with a single column
A bag of words representation of a document corpus, similar to the
format required for `dataset`. This will be used to monitor model
performance during training. Each document in the provided validation
set is randomly split: the first portion is used estimate which topic
each document belongs to, and the second portion is used to estimate
the model's performance at predicting the unseen words in the test data.
method : {'cgs', 'alias'}, optional
The algorithm used for learning the model.
- *cgs:* Collapsed Gibbs sampling
- *alias:* AliasLDA method.
Returns
-------
out : TopicModel
A fitted topic model. This can be used with
:py:func:`~TopicModel.get_topics()` and
:py:func:`~TopicModel.predict()`. While fitting is in progress, several
metrics are shown, including:
+------------------+---------------------------------------------------+
| Field | Description |
+==================+===================================================+
| Elapsed Time | The number of elapsed seconds. |
+------------------+---------------------------------------------------+
| Tokens/second | The number of unique words processed per second |
+------------------+---------------------------------------------------+
| Est. Perplexity | An estimate of the model's ability to model the |
| | training data. See the documentation on evaluate. |
+------------------+---------------------------------------------------+
See Also
--------
TopicModel, TopicModel.get_topics, TopicModel.predict,
turicreate.SArray.dict_trim_by_keys, TopicModel.evaluate
References
----------
- `Wikipedia - Latent Dirichlet allocation
<http://en.wikipedia.org/wiki/Latent_Dirichlet_allocation>`_
- Alias method: Li, A. et al. (2014) `Reducing the Sampling Complexity of
Topic Models. <http://www.sravi.org/pubs/fastlda-kdd2014.pdf>`_.
KDD 2014.
Examples
--------
The following example includes an SArray of documents, where
each element represents a document in "bag of words" representation
-- a dictionary with word keys and whose values are the number of times
that word occurred in the document:
>>> docs = turicreate.SArray('https://static.turi.com/datasets/nytimes')
Once in this form, it is straightforward to learn a topic model.
>>> m = turicreate.topic_model.create(docs)
It is also easy to create a new topic model from an old one -- whether
it was created using Turi Create or another package.
>>> m2 = turicreate.topic_model.create(docs, initial_topics=m['topics'])
To manually fix several words to always be assigned to a topic, use
the `associations` argument. The following will ensure that topic 0
has the most probability for each of the provided words:
>>> from turicreate import SFrame
>>> associations = SFrame({'word':['hurricane', 'wind', 'storm'],
'topic': [0, 0, 0]})
>>> m = turicreate.topic_model.create(docs,
associations=associations)
More advanced usage allows you to control aspects of the model and the
learning method.
>>> import turicreate as tc
>>> m = tc.topic_model.create(docs,
num_topics=20, # number of topics
num_iterations=10, # algorithm parameters
alpha=.01, beta=.1) # hyperparameters
To evaluate the model's ability to generalize, we can create a train/test
split where a portion of the words in each document are held out from
training.
>>> train, test = tc.text_analytics.random_split(.8)
>>> m = tc.topic_model.create(train)
>>> results = m.evaluate(test)
>>> print results['perplexity']
In [29]:
help(tc.text_analytics.random_split)
Help on function random_split in module turicreate.toolkits.text_analytics._util:
random_split(dataset, prob=0.5)
Utility for performing a random split for text data that is already in
bag-of-words format. For each (word, count) pair in a particular element,
the counts are uniformly partitioned in either a training set or a test
set.
Parameters
----------
dataset : SArray of type dict, SFrame with columns of type dict
A data set in bag-of-words format.
prob : float, optional
Probability for sampling a word to be placed in the test set.
Returns
-------
train, test : SArray
Two data sets in bag-of-words format, where the combined counts are
equal to the counts in the original data set.
Examples
--------
>>> docs = turicreate.SArray([{'are':5, 'you':3, 'not': 1, 'entertained':10}])
>>> train, test = turicreate.text_analytics.random_split(docs)
>>> print(train)
[{'not': 1.0, 'you': 3.0, 'are': 3.0, 'entertained': 7.0}]
>>> print(test)
[{'are': 2.0, 'entertained': 3.0}]
In [30]:
train, test = tc.text_analytics.random_split(docs, .8)
In [31]:
m = tc.topic_model.create(train,
num_topics=100, # number of topics
num_iterations=100, # algorithm parameters
alpha=None, beta=.1) # hyperparameters
Learning a topic model
Number of documents 72269
Vocabulary size 108205
Running collapsed Gibbs sampling
+-----------+---------------+----------------+-----------------+
| Iteration | Elapsed Time | Tokens/Second | Est. Perplexity |
+-----------+---------------+----------------+-----------------+
| 10 | 1.66s | 6.17013e+06 | 0 |
| 20 | 3.12s | 6.57117e+06 | 0 |
| 30 | 4.58s | 6.41968e+06 | 0 |
| 40 | 6.00s | 6.61674e+06 | 0 |
| 50 | 7.42s | 6.53873e+06 | 0 |
| 60 | 8.85s | 6.46591e+06 | 0 |
| 70 | 10.38s | 5.92867e+06 | 0 |
| 80 | 11.90s | 6.36375e+06 | 0 |
| 90 | 13.42s | 6.20572e+06 | 0 |
| 100 | 15.03s | 5.4879e+06 | 0 |
+-----------+---------------+----------------+-----------------+
In [32]:
results = m.evaluate(test)
print(results['perplexity'])
4552.105441414741
In [33]:
m
Out[33]:
Class : TopicModel
Schema
------
Vocabulary Size : 108205
Settings
--------
Number of Topics : 100
alpha : 0.5
beta : 0.1
Iterations : 100
Training time : 16.0432
Verbose : True
Accessible fields :
m.topics : An SFrame containing the topics.
m.vocabulary : An SArray containing the words in the vocabulary.
Useful methods :
m.get_topics() : Get the most probable words per topic.
m.predict(new_docs) : Make predictions for new documents.
In [34]:
m.get_topics()
Out[34]:
topic
word
score
0
years
0.004647514462204498
0
evans
0.004059221492305195
0
lebanon
0.0028172696669622205
0
green
0.0028172696669622205
0
time
0.0020982449259741827
1
national
0.005351237598960527
1
back
0.002278117379832135
1
baldwin
0.0018772756121197358
1
chicago
0.0018104686508343358
1
private
0.0016100477669781365
[500 rows x 3 columns]
Note: Only the head of the SFrame is printed.
You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.
In [35]:
help(m.get_topics)
Help on method get_topics in module turicreate.toolkits.topic_model.topic_model:
get_topics(topic_ids=None, num_words=5, cdf_cutoff=1.0, output_type='topic_probabilities') method of turicreate.toolkits.topic_model.topic_model.TopicModel instance
Get the words associated with a given topic. The score column is the
probability of choosing that word given that you have chosen a
particular topic.
Parameters
----------
topic_ids : list of int, optional
The topics to retrieve words. Topic ids are zero-based.
Throws an error if greater than or equal to m['num_topics'], or
if the requested topic name is not present.
num_words : int, optional
The number of words to show.
cdf_cutoff : float, optional
Allows one to only show the most probable words whose cumulative
probability is below this cutoff. For example if there exist
three words where
.. math::
p(word_1 | topic_k) = .1
p(word_2 | topic_k) = .2
p(word_3 | topic_k) = .05
then setting :math:`cdf_{cutoff}=.3` would return only
:math:`word_1` and :math:`word_2` since
:math:`p(word_1 | topic_k) + p(word_2 | topic_k) <= cdf_{cutoff}`
output_type : {'topic_probabilities' | 'topic_words'}, optional
Determine the type of desired output. See below.
Returns
-------
out : SFrame
If output_type is 'topic_probabilities', then the returned value is
an SFrame with a column of words ranked by a column of scores for
each topic. Otherwise, the returned value is a SArray where
each element is a list of the most probable words for each topic.
Examples
--------
Get the highest ranked words for all topics.
>>> docs = turicreate.SArray('https://static.turi.com/datasets/nips-text')
>>> m = turicreate.topic_model.create(docs,
num_iterations=50)
>>> m.get_topics()
+-------+----------+-----------------+
| topic | word | score |
+-------+----------+-----------------+
| 0 | cell | 0.028974400831 |
| 0 | input | 0.0259470208503 |
| 0 | image | 0.0215721599763 |
| 0 | visual | 0.0173635081992 |
| 0 | object | 0.0172447874156 |
| 1 | function | 0.0482834508265 |
| 1 | input | 0.0456270024091 |
| 1 | point | 0.0302662839454 |
| 1 | result | 0.0239474934631 |
| 1 | problem | 0.0231750116011 |
| ... | ... | ... |
+-------+----------+-----------------+
Get the highest ranked words for topics 0 and 1 and show 15 words per
topic.
>>> m.get_topics([0, 1], num_words=15)
+-------+----------+------------------+
| topic | word | score |
+-------+----------+------------------+
| 0 | cell | 0.028974400831 |
| 0 | input | 0.0259470208503 |
| 0 | image | 0.0215721599763 |
| 0 | visual | 0.0173635081992 |
| 0 | object | 0.0172447874156 |
| 0 | response | 0.0139740298286 |
| 0 | layer | 0.0122585145062 |
| 0 | features | 0.0115343177265 |
| 0 | feature | 0.0103530459301 |
| 0 | spatial | 0.00823387994361 |
| ... | ... | ... |
+-------+----------+------------------+
If one wants to instead just get the top words per topic, one may
change the format of the output as follows.
>>> topics = m.get_topics(output_type='topic_words')
dtype: list
Rows: 10
[['cell', 'image', 'input', 'object', 'visual'],
['algorithm', 'data', 'learning', 'method', 'set'],
['function', 'input', 'point', 'problem', 'result'],
['model', 'output', 'pattern', 'set', 'unit'],
['action', 'learning', 'net', 'problem', 'system'],
['error', 'function', 'network', 'parameter', 'weight'],
['information', 'level', 'neural', 'threshold', 'weight'],
['control', 'field', 'model', 'network', 'neuron'],
['hidden', 'layer', 'system', 'training', 'vector'],
['component', 'distribution', 'local', 'model', 'optimal']]
In [37]:
topics = m.get_topics(num_words=10).unstack(['word','score'], \
new_column_name='topic_words')['topic_words'].apply(lambda x: x.keys())
for topic in topics:
print(topic)
['storm', 'florida', 'area', 'due', 'texas', 'people', 'damage', 'hurricane', 'tropical', 'system']
['project', 'ryan', 'founded', 'harvard', 'carroll', 'including', 'wilson', 'school', 'oxford', 'national']
['alliance', 'organization', 'membership', 'board', 'member', 'members', 'association', 'groups', 'group', 'society']
['summer', 'george', 'kan', 'julian', '1978', 'date', 'years', 'wolfe', 'london', 'italian']
['german', 'division', 'forces', 'british', 'army', 'men', 'battle', 'general', 'war', 'military']
['arrow', 'williams', 'california', 'warren', 'santa', 'los', 'san', 'great', 'angeles', 'renamed']
['miller', 'time', 'produced', 'years', 'lewis', 'hamilton', 'morris', '1999', '2005', 'epic']
['played', 'games', 'coach', 'won', 'points', 'team', 'game', 'record', 'season', 'teams']
['green', 'time', 'connecticut', 'evans', 'years', 'sixth', 'lebanon', '2001', 'worked', '2005']
['van', 'berlin', 'im', 'des', 'von', 'die', 'den', 'der', 'das', 'und']
['clothing', 'hat', 'worn', 'wear', 'made', 'green', 'summer', 'black', 'top', 'dress']
['2009', 'year', 'kids', 'october', '2010', '2007', 'school', '2006', '2008', 'national']
['magazine', 'grant', 'oregon', 'portland', 'college', '1985', '2006', 'year', 'long', 'beer']
['network', 'show', 'local', 'news', 'broadcast', 'radio', 'channel', 'stations', 'television', 'station']
['bridge', 'north', 'river', 'county', 'state', 'west', 'route', 'city', 'east', 'road']
['part', 'degree', 'na', 'valid', 'agent', 'history', 'reform', 'born', 'agency', 'trn']
['children', 'father', 'mother', 'married', 'years', 'family', 'born', 'life', 'died', 'time']
['early', 'population', 'century', 'american', 'war', 'government', 'states', 'british', 'united', 'people']
['stories', 'magazine', 'writing', 'history', 'work', 'wrote', 'works', 'published', 'book', 'books']
['collection', 'painting', 'arts', 'museum', 'work', 'york', 'design', 'artists', 'art', 'works']
['named', 'flag', 'orange', 'blue', 'colors', 'yellow', 'tiger', 'black', 'red', 'white']
['made', 'mission', 'test', 'project', 'space', 'aircraft', 'wright', 'design', 'launch', 'flight']
['alan', 'phillips', 'preov', 'koice', 'beth', 'trenn', 'phillip', 'nitra', 'town', 'trnava']
['racing', 'cars', 'car', 'engines', 'models', 'series', 'engine', 'system', 'race', 'model']
['building', 'duck', 'london', 'philadelphia', 'represented', 'major', 'york', 'design', 'journal', 'duncan']
['english', 'language', 'form', 'meaning', 'word', 'words', 'names', 'written', 'languages', 'common']
['gene', 'enzyme', 'dna', 'site', 'proteins', 'protein', 'cells', 'cell', 'structure', 'genes']
['work', 'year', '2009', 'andrew', 'list', 'henin', 'link', 'dates', 'part', 'calendar']
['law', 'public', 'united', 'state', 'legal', 'act', 'states', 'court', 'case', 'federal']
['trains', 'west', 'line', 'rail', 'train', 'station', 'services', 'service', 'opened', 'railway']
['served', 'cemetery', 'received', 'texas', 'po', 'christi', 'university', 'post', 'corpus', 'mexico']
['international', 'worked', 'festival', 'director', 'film', 'awards', 'won', 'award', 'awarded', 'academy']
['empire', 'china', 'chinese', 'dynasty', 'roman', 'greek', 'time', 'king', 'bc', 'emperor']
['port', 'ship', 'coast', 'island', 'sea', 'fleet', 'region', 'bay', 'ships', 'islands']
['south', 'north', 'river', 'mountain', 'water', 'creek', 'lake', 'valley', 'park', 'area']
['january', 'day', 'june', '2009', 'october', 'december', '2010', '2007', '2008', 'september']
['round', 'team', 'won', 'event', 'title', 'world', 'match', 'championship', 'lost', 'open']
['created', '2010', 'ash', 'complex', 'sanders', 'hotel', 'skating', 'school', 'house', 'msn']
['left', 'easy', 'leg', 'ancient', 'benson', 'school', '2003', 'critical', 'british', 'westfield']
['released', 'world', 'games', 'version', 'series', 'video', 'player', 'game', 'players', '2']
['won', 'cup', 'team', 'teams', 'season', 'final', 'football', 'club', 'league', 'played']
['1988', 'called', 'list', '1971', '2010', '1994', 'made', 'trent', 'found', 'local']
['world', 'received', 'usa', 'wall', 'part', 'start0', 'williams', 'work', 'events', 'plotdata']
['years', 'age', 'income', 'median', '18', 'living', 'males', 'town', 'average', 'population']
['youth', 'bamboo', 'ross', 'griffin', 'williams', 'meeting', 'monument', 'town', 'joined', 'girl']
['south', 'european', 'international', 'economic', 'states', 'development', 'united', 'government', 'countries', 'world']
['national', 'constituencies', 'animation', 'part', 'track', 'baldwin', '1982', 'private', 'back', 'chicago']
['hill', 'morgan', 'davis', 'cj', '1992', 'arch', 'child', '1995', 'tony', 'part']
['form', 'called', '1', 'theory', 'set', 'number', 'function', 'type', 'system', 'model']
['form', 'women', 'social', 'life', 'society', 'people', 'time', 'world', 'human', 'work']
['city', 'san', 'paris', 'el', 'france', 'la', 'de', 'french', 'spain', 'spanish']
['city', 'located', 'house', 'park', 'building', 'street', 'built', 'town', 'area', 'century']
['made', 'include', 'variety', 'traditional', 'rice', 'popular', 'food', 'called', 'served', 'wine']
['canadian', 'ontario', 'alberta', 'british', 'york', '2002', 'toronto', 'quebec', 'canada', 'montreal']
['khan', 'afghanistan', 'indian', 'singh', 'temple', 'punjab', 'pakistan', 'sri', 'india', 'community']
['brooklyn', 'city', 'class', 'ambassador', '2004', 'due', 'babylon', 'historical', 'area', '5']
['heat', 'iron', 'high', 'process', 'water', 'form', 'gas', 'temperature', 'nuclear', 'energy']
['airlines', 'regional', 'airport', '2010', 'international', 'travel', 'airways', '2005', 'airline', 'joined']
['head', '15', 'home', '2010', '2003', 'elected', 'park', 'stone', 'gaol', 'numerous']
['poland', 'years', 'polish', 'soviet', 'moscow', 'war', 'russian', 'swedish', 'union', 'russia']
['william', 'henry', 'england', 'sir', 'london', 'royal', 'lord', 'king', 'son', 'john']
['ohio', 'virginia', 'district', 'washington', 'john', 'carolina', 'county', 'state', 'served', 'south']
['state', 'election', 'political', 'minister', 'council', 'national', 'party', 'president', 'government', 'elected']
['district', 'districts', 'register', 'properties', 'places', 'illinois', 'national', 'county', 'historic', 'school']
['operations', 'aircraft', 'base', 'united', 'war', 'service', 'air', 'group', 'force', 'training']
['number', '25', '2010', 'golf', 'time', 'scrooge', 'group', 'university', 'chams', 'world']
['light', 'range', 'made', 'time', 'system', 'small', 'power', 'energy', 'high', 'current']
['1983', 'university', 'states', 'served', 'appointed', '1986', 'degree', 'bill', 'professor', '2000']
['years', 'head', 'joseph', 'school', 'smiths', 'rejoice', 'smith', 'clark', 'stone', 'plates']
['found', 'occur', 'body', 'small', 'disease', 'large', 'species', 'family', 'order', 'birds']
['number', 'fat', '2006', 'named', 'work', 'death', 'published', 'flores', 'vincent', 'villa']
['company', 'business', 'mine', 'sold', 'oil', 'stores', 'industry', 'year', 'production', 'mining']
['martin', 'kitty', 'obesity', 'turtle', 'point', '2000', 'camp', 'clark', 'moved', 'born']
['1994', '1998', 'ryo', 'venezuela', 'tannins', 'national', 'group', 'minor', 'christmas', 'world']
['parish', 'saint', 'bishop', 'church', 'roman', 'council', 'st', 'century', 'catholic', 'pope']
['released', 'records', 'single', 'album', 'songs', 'music', 'band', 'live', 'song', 'tour']
['wales', 'australia', 'played', 'cricket', 'zealand', 'day', 'sydney', 'made', 'australian', 'south']
['information', 'network', 'system', 'software', 'systems', 'internet', 'users', 'technology', 'computer', 'data']
['philippine', 'clara', 'foundation', 'chief', 'full', 'fair', 'philippines', 'manila', 'rupiah', 'belle']
['group', 'began', 'anderson', 'minnesota', 'john', 'period', 'jones', 'te', 'td', 'dont']
['northern', 'cork', 'dublin', 'ireland', 'irish', 'senior', 'john', 'county', 'title', 'medal']
['school', '2006', 'stone', 'york', 'bay', 'village', 'held', 'created', 'local', 'community']
['years', 'jersey', 'police', 'city', 'york', 'family', 'crime', 'gang', 'prison', 'arrested']
['2001', 'american', 'saudi', 'bin', 'world', 'al', 'sh', 'including', 'laden', 'joined']
['school', 'education', 'year', 'students', 'research', 'schools', 'college', 'high', 'university', 'program']
['events', 'time', 'mr', 'ride', 'roller', 'part', 'including', 'bicycle', 'riders', 'bike']
['montenegro', 'yugoslavia', 'serbia', 'school', 'serbian', 'albanian', 'albania', 'croatian', 'croatia', 'year']
['austin', 'shells', 'place', 'peter', 'shell', 'uk', 'school', '2010', 'active', 'mark']
['york', 'series', 'masters', 'booth', 'world', 'free', '2003', 'major', 'birmingham', 'setting']
['role', 'appeared', 'films', 'episode', 'television', 'show', 'series', 'character', 'movie', 'film']
['medicine', 'dr', 'training', 'care', 'hospital', 'health', 'surgery', 'center', 'dog', 'medical']
['norwegian', 'travis', '2007', 'ste', 'bar', 'city', 'home', 'norway', 'parker', 'include']
['jewish', 'christian', 'people', 'god', 'gods', 'religious', 'great', 'church', 'churches', 'jesus']
['zion', 'contest', 'miss', 'relay', 'gold', 'post', 'nec', 'mori', 'yacht', 'time']
['york', 'theatre', 'theater', 'dance', 'musical', 'orchestra', 'opera', 'music', 'performed', 'festival']
['tax', 'money', 'financial', 'market', 'bank', 'million', 'business', 'price', 'services', 'company']
['life', 'death', 'father', 'end', 'find', 'back', 'man', 'make', 'time', 'tells']
['years', '2007', 'walker', 'post', '1911', 'croydon', 'burma', 'burmese', 'named', 'day']
['high', 'vegas', 'purchased', 'alice', 'paul', 'hotel', 'scott', 'ranch', 'las', 'highlands']
['horse', 'stakes', 'born', 'american', 'horses', 'race', 'boas', 'breed', 'english', 'racing']
In [40]:
help(m)
Help on TopicModel in module turicreate.toolkits.topic_model.topic_model object:
class TopicModel(turicreate.toolkits._model.Model)
| TopicModel objects can be used to predict the underlying topic of a
| document.
|
| This model cannot be constructed directly. Instead, use
| :func:`turicreate.topic_model.create` to create an instance
| of this model. A detailed list of parameter options and code samples
| are available in the documentation for the create function.
|
| Method resolution order:
| TopicModel
| turicreate.toolkits._model.Model
| turicreate.toolkits._model.ExposeAttributesFromProxy
| builtins.object
|
| Methods defined here:
|
| __init__(self, model_proxy)
| Initialize self. See help(type(self)) for accurate signature.
|
| __repr__(self)
| Print a string description of the model when the model name is entered
| in the terminal.
|
| __str__(self)
| Return a string description of the model to the ``print`` method.
|
| Returns
| -------
| out : string
| A description of the model.
|
| evaluate(self, train_data, test_data=None, metric='perplexity')
| Estimate the model's ability to predict new data. Imagine you have a
| corpus of books. One common approach to evaluating topic models is to
| train on the first half of all of the books and see how well the model
| predicts the second half of each book.
|
| This method returns a metric called perplexity, which is related to the
| likelihood of observing these words under the given model. See
| :py:func:`~turicreate.topic_model.perplexity` for more details.
|
| The provided `train_data` and `test_data` must have the same length,
| i.e., both data sets must have the same number of documents; the model
| will use train_data to estimate which topic the document belongs to, and
| this is used to estimate the model's performance at predicting the
| unseen words in the test data.
|
| See :py:func:`~turicreate.topic_model.TopicModel.predict` for details
| on how these predictions are made, and see
| :py:func:`~turicreate.text_analytics.random_split` for a helper function
| that can be used for making train/test splits.
|
| Parameters
| ----------
| train_data : SArray or SFrame
| A set of documents to predict topics for.
|
| test_data : SArray or SFrame, optional
| A set of documents to evaluate performance on.
| By default this will set to be the same as train_data.
|
| metric : str
| The chosen metric to use for evaluating the topic model.
| Currently only 'perplexity' is supported.
|
| Returns
| -------
| out : dict
| The set of estimated evaluation metrics.
|
| See Also
| --------
| predict, turicreate.toolkits.text_analytics.random_split
|
| Examples
| --------
| >>> docs = turicreate.SArray('https://static.turi.com/datasets/nips-text')
| >>> train_data, test_data = turicreate.text_analytics.random_split(docs)
| >>> m = turicreate.topic_model.create(train_data)
| >>> m.evaluate(train_data, test_data)
| {'perplexity': 2467.530370396021}
|
| get_topics(self, topic_ids=None, num_words=5, cdf_cutoff=1.0, output_type='topic_probabilities')
| Get the words associated with a given topic. The score column is the
| probability of choosing that word given that you have chosen a
| particular topic.
|
| Parameters
| ----------
| topic_ids : list of int, optional
| The topics to retrieve words. Topic ids are zero-based.
| Throws an error if greater than or equal to m['num_topics'], or
| if the requested topic name is not present.
|
| num_words : int, optional
| The number of words to show.
|
| cdf_cutoff : float, optional
| Allows one to only show the most probable words whose cumulative
| probability is below this cutoff. For example if there exist
| three words where
|
| .. math::
| p(word_1 | topic_k) = .1
|
| p(word_2 | topic_k) = .2
|
| p(word_3 | topic_k) = .05
|
| then setting :math:`cdf_{cutoff}=.3` would return only
| :math:`word_1` and :math:`word_2` since
| :math:`p(word_1 | topic_k) + p(word_2 | topic_k) <= cdf_{cutoff}`
|
| output_type : {'topic_probabilities' | 'topic_words'}, optional
| Determine the type of desired output. See below.
|
| Returns
| -------
| out : SFrame
| If output_type is 'topic_probabilities', then the returned value is
| an SFrame with a column of words ranked by a column of scores for
| each topic. Otherwise, the returned value is a SArray where
| each element is a list of the most probable words for each topic.
|
| Examples
| --------
| Get the highest ranked words for all topics.
|
| >>> docs = turicreate.SArray('https://static.turi.com/datasets/nips-text')
| >>> m = turicreate.topic_model.create(docs,
| num_iterations=50)
| >>> m.get_topics()
| +-------+----------+-----------------+
| | topic | word | score |
| +-------+----------+-----------------+
| | 0 | cell | 0.028974400831 |
| | 0 | input | 0.0259470208503 |
| | 0 | image | 0.0215721599763 |
| | 0 | visual | 0.0173635081992 |
| | 0 | object | 0.0172447874156 |
| | 1 | function | 0.0482834508265 |
| | 1 | input | 0.0456270024091 |
| | 1 | point | 0.0302662839454 |
| | 1 | result | 0.0239474934631 |
| | 1 | problem | 0.0231750116011 |
| | ... | ... | ... |
| +-------+----------+-----------------+
|
| Get the highest ranked words for topics 0 and 1 and show 15 words per
| topic.
|
| >>> m.get_topics([0, 1], num_words=15)
| +-------+----------+------------------+
| | topic | word | score |
| +-------+----------+------------------+
| | 0 | cell | 0.028974400831 |
| | 0 | input | 0.0259470208503 |
| | 0 | image | 0.0215721599763 |
| | 0 | visual | 0.0173635081992 |
| | 0 | object | 0.0172447874156 |
| | 0 | response | 0.0139740298286 |
| | 0 | layer | 0.0122585145062 |
| | 0 | features | 0.0115343177265 |
| | 0 | feature | 0.0103530459301 |
| | 0 | spatial | 0.00823387994361 |
| | ... | ... | ... |
| +-------+----------+------------------+
|
| If one wants to instead just get the top words per topic, one may
| change the format of the output as follows.
|
| >>> topics = m.get_topics(output_type='topic_words')
| dtype: list
| Rows: 10
| [['cell', 'image', 'input', 'object', 'visual'],
| ['algorithm', 'data', 'learning', 'method', 'set'],
| ['function', 'input', 'point', 'problem', 'result'],
| ['model', 'output', 'pattern', 'set', 'unit'],
| ['action', 'learning', 'net', 'problem', 'system'],
| ['error', 'function', 'network', 'parameter', 'weight'],
| ['information', 'level', 'neural', 'threshold', 'weight'],
| ['control', 'field', 'model', 'network', 'neuron'],
| ['hidden', 'layer', 'system', 'training', 'vector'],
| ['component', 'distribution', 'local', 'model', 'optimal']]
|
| predict(self, dataset, output_type='assignment', num_burnin=None)
| Use the model to predict topics for each document. The provided
| `dataset` should be an SArray object where each element is a dict
| representing a single document in bag-of-words format, where keys
| are words and values are their corresponding counts. If `dataset` is
| an SFrame, then it must contain a single column of dict type.
|
| The current implementation will make inferences about each document
| given its estimates of the topics learned when creating the model.
| This is done via Gibbs sampling.
|
| Parameters
| ----------
| dataset : SArray, SFrame of type dict
| A set of documents to use for making predictions.
|
| output_type : str, optional
| The type of output desired. This can either be
|
| - assignment: the returned values are integers in [0, num_topics)
| - probability: each returned prediction is a vector with length
| num_topics, where element k represents the probability that
| document belongs to topic k.
|
| num_burnin : int, optional
| The number of iterations of Gibbs sampling to perform when
| inferring the topics for documents at prediction time.
| If provided this will override the burnin value set during
| training.
|
| Returns
| -------
| out : SArray
|
| See Also
| --------
| evaluate
|
| Examples
| --------
| Make predictions about which topic each document belongs to.
|
| >>> docs = turicreate.SArray('https://static.turi.com/datasets/nips-text')
| >>> m = turicreate.topic_model.create(docs)
| >>> pred = m.predict(docs)
|
| If one is interested in the probability of each topic
|
| >>> pred = m.predict(docs, output_type='probability')
|
| Notes
| -----
| For each unique word w in a document d, we sample an assignment to
| topic k with probability proportional to
|
| .. math::
| p(z_{dw} = k) \propto (n_{d,k} + \alpha) * \Phi_{w,k}
|
| where
|
| - :math:`W` is the size of the vocabulary,
| - :math:`n_{d,k}` is the number of other times we have assigned a word in
| document to d to topic :math:`k`,
| - :math:`\Phi_{w,k}` is the probability under the model of choosing word
| :math:`w` given the word is of topic :math:`k`. This is the matrix
| returned by calling `m['topics']`.
|
| This represents a collapsed Gibbs sampler for the document assignments
| while we keep the topics learned during training fixed.
| This process is done in parallel across all documents, five times per
| document.
|
| ----------------------------------------------------------------------
| Methods inherited from turicreate.toolkits._model.Model:
|
| save(self, location)
| Save the model. The model is saved as a directory which can then be
| loaded using the :py:func:`~turicreate.load_model` method.
|
| Parameters
| ----------
| location : string
| Target destination for the model. Can be a local path or remote URL.
|
| See Also
| ----------
| turicreate.load_model
|
| Examples
| ----------
| >>> model.save('my_model_file')
| >>> loaded_model = turicreate.load_model('my_model_file')
|
| summary(self, output=None)
| Print a summary of the model. The summary includes a description of
| training data, options, hyper-parameters, and statistics measured
| during model creation.
|
| Parameters
| ----------
| output : str, None
| The type of summary to return.
|
| - None or 'stdout' : print directly to stdout.
|
| - 'str' : string of summary
|
| - 'dict' : a dict with 'sections' and 'section_titles' ordered
| lists. The entries in the 'sections' list are tuples of the form
| ('label', 'value').
|
| Examples
| --------
| >>> m.summary()
|
| ----------------------------------------------------------------------
| Methods inherited from turicreate.toolkits._model.ExposeAttributesFromProxy:
|
| __dir__(self)
| Combine the results of dir from the current class with the results of
| list_fields().
|
| __getattribute__(self, attr)
| Use the internal proxy object for obtaining list_fields.
|
| ----------------------------------------------------------------------
| Data descriptors inherited from turicreate.toolkits._model.ExposeAttributesFromProxy:
|
| __dict__
| dictionary for instance variables (if defined)
|
| __weakref__
| list of weak references to the object (if defined)
|
| ----------------------------------------------------------------------
| Data and other attributes inherited from turicreate.toolkits._model.ExposeAttributesFromProxy:
|
| __proxy__ = None
In [41]:
def print_topics(m):
topics = m.get_topics(num_words=5)
topics = topics.unstack(['word','score'], new_column_name='topic_words')['topic_words']
topics = topics.apply(lambda x: x.keys())
for topic in topics:
print(topic)
print_topics(m)
['people', 'texas', 'tropical', 'florida', 'storm']
['school', 'founded', 'wilson', 'harvard', 'including']
['members', 'association', 'society', 'member', 'group']
['summer', 'date', 'julian', 'years', 'george']
['war', 'general', 'battle', 'german', 'army']
['california', 'angeles', 'san', 'los', 'santa']
['miller', 'morris', '2005', '1999', 'time']
['game', 'season', 'team', 'points', 'games']
['evans', 'years', 'time', 'green', 'lebanon']
['und', 'der', 'die', 'van', 'von']
['wear', 'worn', 'made', 'green', 'top']
['2008', 'year', '2007', '2009', '2010']
['oregon', 'magazine', 'year', 'portland', 'beer']
['network', 'news', 'radio', 'show', 'station']
['bridge', 'north', 'route', 'west', 'road']
['part', 'born', 'na', 'agent', 'valid']
['years', 'family', 'father', 'died', 'time']
['united', 'population', 'people', 'american', 'states']
['work', 'magazine', 'published', 'book', 'books']
['collection', 'museum', 'arts', 'art', 'work']
['white', 'black', 'red', 'flag', 'blue']
['aircraft', 'design', 'space', 'project', 'flight']
['trenn', 'phillip', 'nitra', 'koice', 'preov']
['engine', 'cars', 'car', 'race', 'model']
['london', 'duck', 'journal', 'philadelphia', 'york']
['english', 'word', 'language', 'languages', 'words']
['dna', 'cell', 'cells', 'enzyme', 'protein']
['2009', 'calendar', 'year', 'link', 'list']
['court', 'act', 'state', 'states', 'law']
['station', 'services', 'railway', 'line', 'service']
['cemetery', 'university', 'mexico', 'texas', 'corpus']
['film', 'worked', 'festival', 'award', 'awards']
['chinese', 'emperor', 'bc', 'king', 'empire']
['islands', 'region', 'ships', 'island', 'ship']
['water', 'park', 'lake', 'area', 'river']
['2007', '2009', 'december', '2010', '2008']
['match', 'world', 'title', 'won', 'championship']
['msn', 'house', '2010', 'skating', 'school']
['school', 'westfield', 'easy', 'british', '2003']
['player', 'players', 'game', 'games', 'series']
['team', 'league', 'season', 'club', 'football']
['local', 'list', 'found', 'called', '2010']
['plotdata', 'received', 'world', 'part', 'usa']
['income', 'population', '18', 'years', 'age']
['youth', 'ross', 'griffin', 'joined', 'williams']
['south', 'government', 'international', 'countries', 'world']
['national', 'private', 'back', 'chicago', 'baldwin']
['cj', 'hill', 'morgan', 'davis', 'child']
['set', '1', 'model', 'theory', 'number']
['social', 'people', 'time', 'form', 'work']
['french', 'de', 'spanish', 'france', 'la']
['area', 'city', 'building', 'town', 'built']
['food', 'popular', 'made', 'called', 'wine']
['canadian', 'british', 'ontario', 'canada', 'toronto']
['india', 'temple', 'khan', 'pakistan', 'indian']
['babylon', '2004', 'area', 'class', '5']
['water', 'process', 'nuclear', 'energy', 'form']
['international', '2010', 'airport', 'airlines', 'airline']
['15', 'park', 'head', 'stone', '2003']
['soviet', 'russia', 'poland', 'russian', 'polish']
['john', 'royal', 'william', 'sir', 'king']
['virginia', 'county', 'served', 'state', 'carolina']
['election', 'government', 'president', 'state', 'party']
['school', 'county', 'national', 'districts', 'district']
['group', 'war', 'force', 'air', 'aircraft']
['world', 'university', '2010', 'chams', 'number']
['energy', 'power', 'system', 'time', 'light']
['bill', 'professor', 'degree', 'served', 'university']
['joseph', 'head', 'smiths', 'smith', 'school']
['birds', 'family', 'small', 'species', 'found']
['fat', '2006', 'named', 'work', 'vincent']
['business', 'stores', 'oil', 'company', 'mine']
['2000', 'kitty', 'camp', 'moved', 'obesity']
['1994', 'christmas', 'world', 'national', 'minor']
['bishop', 'saint', 'church', 'st', 'century']
['band', 'released', 'album', 'music', 'song']
['made', 'zealand', 'australian', 'australia', 'south']
['system', 'software', 'data', 'systems', 'information']
['full', 'fair', 'philippines', 'manila', 'philippine']
['began', 'jones', 'minnesota', 'anderson', 'td']
['county', 'ireland', 'medal', 'irish', 'dublin']
['school', 'held', 'village', 'local', 'community']
['family', 'city', 'prison', 'police', 'york']
['al', 'bin', '2001', 'joined', 'world']
['college', 'university', 'education', 'students', 'school']
['including', 'riders', 'ride', 'time', 'part']
['year', 'yugoslavia', 'serbian', 'albanian', 'serbia']
['shells', 'uk', 'school', 'austin', 'shell']
['world', 'birmingham', 'york', 'booth', 'free']
['show', 'role', 'episode', 'series', 'film']
['dr', 'care', 'health', 'medical', 'hospital']
['include', '2007', 'norway', 'norwegian', 'city']
['religious', 'christian', 'church', 'god', 'jesus']
['nec', 'mori', 'miss', 'post', 'time']
['festival', 'theatre', 'dance', 'music', 'opera']
['business', 'company', 'financial', 'bank', 'million']
['man', 'tells', 'life', 'back', 'time']
['years', '2007', 'post', 'croydon', 'day']
['las', 'scott', 'vegas', 'ranch', 'hotel']
['american', 'horses', 'horse', 'breed', 'stakes']
pred = m.predict(another_data)
pred = m.predict(another_data, output_type='probabilities')
In [46]:
dir(m)
Out[46]:
['__class__',
'__delattr__',
'__dict__',
'__dir__',
'__doc__',
'__eq__',
'__format__',
'__ge__',
'__getattribute__',
'__gt__',
'__hash__',
'__init__',
'__le__',
'__lt__',
'__module__',
'__ne__',
'__new__',
'__proxy__',
'__reduce__',
'__reduce_ex__',
'__repr__',
'__setattr__',
'__sizeof__',
'__str__',
'__subclasshook__',
'__weakref__',
'_get',
'_get_queryable_methods',
'_get_summary_struct',
'_list_fields',
'_name',
'_native_name',
'_training_stats',
'alpha',
'beta',
'evaluate',
'get_topics',
'num_burnin',
'num_iterations',
'num_topics',
'predict',
'print_interval',
'save',
'summary',
'topics',
'training_iterations',
'training_time',
'validation_time',
'verbose',
'vocabulary']
In [48]:
m.vocabulary
Out[48]:
dtype: str
Rows: 108205
['book', 'political', 'time', 'readers', 'individual', 'appeared', 'peikoff', 'concepts', '100', 'picture', 'america', 'reviewers', 'philosopher', 'screenwriter', 'work', 'traditional', 'purged', 'ayn', 'york', 'articles', 'pharmacy', 'scholarship', '2001', 'designed', 'permission', 'taking', 'historian', 'library', 'russian', 'extent', 'childhood', 'respondents', 'language', 'alisa', 'writing', 'union', 'libertarian', 'positive', 'jennifer', 'notes', 'line', 'burns', 'state', 'crimea', 'sciabarra', 'based', 'rights', 'life', 'shes', 'argument', 'nonfiction', 'rejection', 'allowed', 'reason', 'culture', 'closest', 'shrugged', 'free', 'january', 'success', 'living', 'robert', 'literary', 'animated', 'american', 'reviews', 'paterson', 'people', 'percent', 'house', 'academic', 'sacrificing', 'referred', 'broadway', 'fountainhead', 'lectures', 'john', 'inspiration', 'conditions', 'lifetime', 'written', '1938', 'established', 'barbara', 'twelve', 'modern', 'final', 'intellectual', 'audience', 'stated', 'selfinterest', 'achievement', 'century', 'relationship', 'allowing', 'delivering', 'writers', 'influence', 'branden', 'film', ... ]
In [50]:
m.topics
Out[50]:
108205
In [51]:
m2 = tc.topic_model.create(docs,
num_topics=100,
initial_topics=m.topics)
Initializing from provided topics and vocabulary.
Learning a topic model
Number of documents 72269
Vocabulary size 171005
Running collapsed Gibbs sampling
+-----------+---------------+----------------+-----------------+
| Iteration | Elapsed Time | Tokens/Second | Est. Perplexity |
+-----------+---------------+----------------+-----------------+
| 10 | 2.99s | 7.251e+06 | 0 |
+-----------+---------------+----------------+-----------------+
In [52]:
associations = tc.SFrame()
associations['word'] = ['recognition']
associations['topic'] = [0]
In [53]:
m2 = tc.topic_model.create(docs,
num_topics=20,
num_iterations=50,
associations=associations,
verbose=False)
Learning a topic model
Number of documents 72269
Vocabulary size 171005
Running collapsed Gibbs sampling
+-----------+---------------+----------------+-----------------+
| Iteration | Elapsed Time | Tokens/Second | Est. Perplexity |
+-----------+---------------+----------------+-----------------+
| 10 | 2.10s | 1.04058e+07 | 0 |
| 20 | 3.97s | 1.09325e+07 | 0 |
| 30 | 5.79s | 9.42067e+06 | 0 |
| 40 | 7.66s | 1.0637e+07 | 0 |
| 50 | 9.51s | 9.86708e+06 | 0 |
+-----------+---------------+----------------+-----------------+
In [54]:
m2.get_topics(num_words=10)
Out[54]:
topic
word
score
0
line
0.0109206038501584
0
german
0.010542910113799127
0
de
0.010148794910641626
0
railway
0.010079824750089063
0
english
0.009242329943379372
0
chinese
0.008900763433976205
0
language
0.008654441432002766
0
china
0.008490226764020474
0
large
0.008004151346792889
0
russian
0.007705280651065117
[200 rows x 3 columns]
Note: Only the head of the SFrame is printed.
You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.
In [55]:
print_topics(m2)
['german', 'line', 'english', 'de', 'railway']
['son', 'time', 'book', 'life', 'john']
['role', 'episode', 'show', 'film', 'series']
['york', 'station', 'park', 'de', 'company']
['information', 'law', 'time', 'work', 'social']
['west', 'county', 'north', 'city', 'east']
['years', '18', 'population', 'town', 'age']
['games', 'team', 'game', 'won', 'season']
['company', 'aircraft', 'air', 'force', 'division']
['india', 'art', 'century', 'roman', 'church']
['government', 'students', 'national', 'state', 'party']
['schools', 'college', 'university', 'school', 'high']
['series', 'world', 'back', 'king', 'time']
['services', 'system', 'service', 'million', 'engine']
['area', 'built', 'river', 'road', 'region']
['systems', 'set', 'number', 'system', 'data']
['water', 'small', 'species', 'food', 'found']
['league', 'year', 'club', 'song', 'time']
['released', 'music', 'songs', 'album', 'band']
['army', 'united', 'states', 'court', 'war']
In [ ]:
Content source: computational-class/cjc
Similar notebooks: