In [15]:

    

import sys, os
import nltk
import pandas as pd


    

In [16]:

    

from collections import defaultdict
from gensim import corpora, models, similarities
def make_dictionary(documents):
    """
    construct a dictionary, i.e. mapping btwn word ids and their freq of occurence in the whole corpus
    filter dictionary to remove stopwords and words occuring < min_count times
    
    input: documents is an iterable consisting of all the words in the corpus 
    output: filtered dictionary
    """

    
    dictionary = corpora.Dictionary(documents)

    stop_words = nltk.corpus.stopwords.words('english') 
    min_count = 2
    stop_ids = [dictionary.token2id[word] for word in stop_words
               if word in dictionary.token2id]
    rare_ids = [id for id, freq in dictionary.dfs.items()
                if freq < min_count]
    dictionary.filter_tokens(stop_ids + rare_ids)
    dictionary.compactify()
    return(dictionary)

def make_corpus(documents):
    """
    """
    dictionary = make_dictionary(documents)
    # convert corpus to vectors using bag-of-words representation, i.e. tuples of word indices and word counts
    corpus = [dictionary.doc2bow(words) for words in documents]
    return(corpus, dictionary)

def make_lsi_similarity_matrix(tfidf_corpus, dictionary):
    """
    construct LSI (latent semantic indexing) model on Tfidf-transformed corpus, print model topics, 
    return similarity matrix.
    """
    # construct model
    lsi = models.lsimodel.LsiModel(tfidf_corpus, id2word=dictionary, num_topics=1000) 
    lsi.save('lsi-model.save')
    # create similarity matrix
    matsim = similarities.MatrixSimilarity(lsi[tfidf_corpus], num_best=1000)
    return(matsim)

def make_lda_similarity_matrix(corpus, dictionary):
    """
    Latent Dirichlet Allocation (LDA) model
    """
    # construct model
    lda = models.ldamodel.LdaModel(corpus, id2word=dictionary, num_topics=100)
    lda.save('lda-model.save')
    # create similarity matrix
    matsim = similarities.MatrixSimilarity(lda[corpus], num_best=1000)
    return(matsim)


    

In [17]:

    

# Read database of data
os.chdir('../data')

output_fname="articles-n-forums-posts.csv"

# Read articles from file
input_fname="AutismParentMagazine-posts-clean.csv"
df=pd.read_csv(input_fname,index_col=0)
df.index.name='post id'
df.head(1)


    

    
Out[17]:







  

    

      

      
title
      
source
      
category
      
text
      
href
    
    

      
post id
      

      

      

      

      

    
  
  

    

      
0
      
Autism, Head Banging and other Self Harming Be...
      
https://www.autismparentingmagazine.com/
      
['category-applied-behavior-analysis-aba']
      
For children with autism spectrum disorder (AS...
      
https://www.autismparentingmagazine.com/autism...
    
  

In [18]:

    

# Read articles from file
input_fname="ehealthforum-posts-clean.csv"
df2=pd.read_csv(input_fname,index_col=0)
df2.index.name='post id'
df2['source']='http://ehealthforum.com'
df2['category']='forums'
df2.head(1)


    

    
Out[18]:







  

    

      

      
title
      
text
      
href
      
user id
      
mother post id
      
source
      
category
    
    

      
post id
      

      

      

      

      

      

      

    
  
  

    

      
1
      
Possible autism signs ?
      
my nephew who is four years has very bad mood ...
      
http://ehealthforum.com/health/possible-autism...
      
55473.0
      
0
      
http://ehealthforum.com
      
forums
    
  

In [19]:

    

input_fname="MedHelp-posts-clean.csv"
df3=pd.read_csv(input_fname,index_col=0)
df3['source']='http://www.medhelp.org'
df3['category']='forums'

#
#Remove questions from forum:
df3=df3.drop(df3.loc[df3.index == df3['mother post id']].index)
del df3['mother post id']
del df3['user id']
df3.head(1)


    

    
Out[19]:







  

    

      

      
title
      
text
      
href
      
source
      
category
    
    

      
post id
      

      

      

      

      

    
  
  

    

      
2
      
Inappropriate Masterbation Down Syndrome
      
A related discussion, self injusry in masturba...
      
http://www.medhelp.org//posts/Autism--Asperger...
      
http://www.medhelp.org
      
forums
    
  

In [20]:

    

# Read articles from file
input_fname="reditt-posts.csv"
df4=pd.read_csv(input_fname,index_col=0)
df4.index.name='post id'
df4['source']='http://www.reditt.com'
df4['category']='forums'
df4.head(1)


    

    
Out[20]:







  

    

      

      
title
      
text
      
href
      
user id
      
source
      
category
    
    

      
post id
      

      

      

      

      

      

    
  
  

    

      
1
      
Sulfur and sulfates question.
      
I have been subbing in a special education cla...
      
http://www.reddit.com/r/autism/comments/1xuenq...
      
0
      
http://www.reditt.com
      
forums
    
  

In [21]:

    

# Join datasets
print(len(df))
df=df.append(df2,ignore_index=True)
print(len(df))
df=df.append(df3,ignore_index=True)
print(len(df))
df=df.append(df4,ignore_index=True)
print(len(df))
del df2
del df3
del df4

# delete mother post id
del df['mother post id']


    

    




212
1272
2249
3776

In [22]:

    

df.index.name='post id'


    

In [23]:

    

# Make shorter version of reference:

def short_ref(href):
    if 'autismparentingmagazine' in href:
        return 'AutismParentingMagazine'
    elif 'medhelp' in href:
        return 'MedHelp'
    elif 'ehealthforum' in href:
        return 'eHealthForum'
    elif 'reddit' in href:
        return 'Reddit'

df['href_short'] = df['href'].apply(short_ref)


    

    
Out[23]:







  

    

      

      
category
      
href
      
source
      
text
      
title
      
user id
      
href_short
    
    

      
post id
      

      

      

      

      

      

      

    
  
  

    

      
0
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/autism...
      
https://www.autismparentingmagazine.com/
      
For children with autism spectrum disorder (AS...
      
Autism, Head Banging and other Self Harming Be...
      
NaN
      
AutismParentingMagazine
    
  

In [29]:

    

df.head(300)


    

    
Out[29]:







  

    

      

      
category
      
href
      
source
      
text
      
title
      
user id
      
href_short
      
tokens
      
text_short
    
    

      
post id
      

      

      

      

      

      

      

      

      

    
  
  

    

      
0
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/autism...
      
https://www.autismparentingmagazine.com/
      
For children with autism spectrum disorder (AS...
      
Autism, Head Banging and other Self Harming Be...
      
NaN
      
AutismParentingMagazine
      
[autism, head, bang, and, other, self, harm, b...
      
For children with autism spectrum disorder (AS...
    
    

      
1
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/high-q...
      
https://www.autismparentingmagazine.com/
      
Dr. Stephen Shore once said “If you’ve met one...
      
High Quality ABA Treatment:  What Every Parent...
      
NaN
      
AutismParentingMagazine
      
[high, quality, aba, treatment, what, every, p...
      
Dr. Stephen Shore once said “If you’ve met one...
    
    

      
2
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/choosi...
      
https://www.autismparentingmagazine.com/
      
Help! I am going to be starting Applied Behav...
      
Help: I Don’t Know How to Choose an Applied Be...
      
NaN
      
AutismParentingMagazine
      
[help, i, don, t, know, how, to, choose, an, a...
      
Help! I am going to be starting Applied Behav...
    
    

      
3
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/help-a...
      
https://www.autismparentingmagazine.com/
      
How do you handle high anxiety of a child on t...
      
HELP: My Autistic Child is Absolutely Terrifie...
      
NaN
      
AutismParentingMagazine
      
[help, my, autistic, child, be, absolutely, te...
      
How do you handle high anxiety of a child on t...
    
    

      
4
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/help-i...
      
https://www.autismparentingmagazine.com/
      
A grandfather from Singapore asks… My eldest g...
      
HELP: I Need Communication Advice for Autistic...
      
NaN
      
AutismParentingMagazine
      
[help, i, need, communication, advice, for, au...
      
A grandfather from Singapore asks… My eldest g...
    
    

      
5
      
['category-applied-behavior-analysis-aba']
      
https://www.autismparentingmagazine.com/help-m...
      
https://www.autismparentingmagazine.com/
      
Hi, I am the parent of a 13-year-old boy with ...
      
Help: My Aspie Lacks Motivation
      
NaN
      
AutismParentingMagazine
      
[help, my, aspie, lack, motivation, hi, i, be,...
      
Hi, I am the parent of a 13-year-old boy with ...
    
    

      
6
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/autism...
      
https://www.autismparentingmagazine.com/
      
It will be some time before we see a positive ...
      
Exciting Campaign Launched to Tackle the Autis...
      
NaN
      
AutismParentingMagazine
      
[excite, campaign, launch, to, tackle, the, au...
      
It will be some time before we see a positive ...
    
    

      
7
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/restau...
      
https://www.autismparentingmagazine.com/
      
Restaurant owner Andrew Iredale could hardly b...
      
How One Restaurateur Makes Eating Out Autism-F...
      
NaN
      
AutismParentingMagazine
      
[how, one, restaurateur, make, eat, out, autis...
      
Restaurant owner Andrew Iredale could hardly b...
    
    

      
8
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/lloyd-...
      
https://www.autismparentingmagazine.com/
      
Autism can be one of the most difficult diagno...
      
Lloyd Claycomb’s Views on Autism
      
NaN
      
AutismParentingMagazine
      
[lloyd, claycomb, s, view, on, autism, autism,...
      
Autism can be one of the most difficult diagno...
    
    

      
9
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/erasin...
      
https://www.autismparentingmagazine.com/
      
An Interview with Dr. Tony Attwood, PhD, AS/AS...
      
Erasing the Prejudice – Are People on the Spec...
      
NaN
      
AutismParentingMagazine
      
[erase, the, prejudice, be, people, on, the, s...
      
An Interview with Dr. Tony Attwood, PhD, AS/AS...
    
    

      
10
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/the-ne...
      
https://www.autismparentingmagazine.com/
      
A civil rights lawyer and autism mom talks abo...
      
The Never Ending Battle for Autism Rights – A ...
      
NaN
      
AutismParentingMagazine
      
[the, never, end, battle, for, autism, right, ...
      
A civil rights lawyer and autism mom talks abo...
    
    

      
11
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/how-on...
      
https://www.autismparentingmagazine.com/
      
In two short years, Tikko has captured the hea...
      
How One Special Bear Changed the Way People Se...
      
NaN
      
AutismParentingMagazine
      
[how, one, special, bear, change, the, way, pe...
      
In two short years, Tikko has captured the hea...
    
    

      
12
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/winnin...
      
https://www.autismparentingmagazine.com/
      
Every time the professional fighters meet thei...
      
Winning the Fight for Autism
      
NaN
      
AutismParentingMagazine
      
[win, the, fight, for, autism, every, time, th...
      
Every time the professional fighters meet thei...
    
    

      
13
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/autism...
      
https://www.autismparentingmagazine.com/
      
Autism Services for Military Families by Lesli...
      
The Right for Autism Services for U.S. Militar...
      
NaN
      
AutismParentingMagazine
      
[the, right, for, autism, service, for, u, s, ...
      
Autism Services for Military Families by Lesli...
    
    

      
14
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/bridge...
      
https://www.autismparentingmagazine.com/
      
Bridge the Gap Gala Raises over $100,000 by Le...
      
Bridge the Gap Gala Raises over $100,000
      
NaN
      
AutismParentingMagazine
      
[bridge, the, gap, gala, raise, over, 100, 000...
      
Bridge the Gap Gala Raises over $100,000 by Le...
    
    

      
15
      
['category-autism-advocacy']
      
https://www.autismparentingmagazine.com/exposi...
      
https://www.autismparentingmagazine.com/
      
We started this magazine to help parents, so...
      
Exposing Autism Speaks – Podcast 4
      
NaN
      
AutismParentingMagazine
      
[expose, autism, speak, podcast, 4, we, start,...
      
We started this magazine to help parents, so...
    
    

      
16
      
['category-autism-and-diet']
      
https://www.autismparentingmagazine.com/gluten...
      
https://www.autismparentingmagazine.com/
      
Recent clinical trials looking at gluten-free ...
      
Top Ways a Gluten Free Diet  Can Help Kids wit...
      
NaN
      
AutismParentingMagazine
      
[top, ways, a, gluten, free, diet, can, help, ...
      
Recent clinical trials looking at gluten-free ...
    
    

      
17
      
['category-autism-and-diet']
      
https://www.autismparentingmagazine.com/asperg...
      
https://www.autismparentingmagazine.com/
      
I find the suggestion that there needs to be a...
      
Asperger’s Syndrome and the Power of a Good Diet
      
NaN
      
AutismParentingMagazine
      
[asperger, s, syndrome, and, the, power, of, a...
      
I find the suggestion that there needs to be a...
    
    

      
18
      
['category-autism-and-diet']
      
https://www.autismparentingmagazine.com/six-st...
      
https://www.autismparentingmagazine.com/
      
“Wow, how did you get him/her to do that?,” pa...
      
Six Steps to a Fun, Friendly, Family Fitness P...
      
NaN
      
AutismParentingMagazine
      
[six, step, to, a, fun, friendly, family, fitn...
      
“Wow, how did you get him/her to do that?,” pa...
    
    

      
19
      
['category-autism-and-diet']
      
https://www.autismparentingmagazine.com/qa-sec...
      
https://www.autismparentingmagazine.com/
      
This is the question of the month as featured ...
      
Q&A Section: How does Gluten free casein free ...
      
NaN
      
AutismParentingMagazine
      
[q, a, section, how, do, gluten, free, casein,...
      
This is the question of the month as featured ...
    
    

      
20
      
['category-autism-and-food']
      
https://www.autismparentingmagazine.com/mallor...
      
https://www.autismparentingmagazine.com/
      
I had the pleasure of reviewing Mallory’s Marv...
      
Mallory’s Marvelous Muffins
      
NaN
      
AutismParentingMagazine
      
[mallory, s, marvelous, muffins, i, have, the,...
      
I had the pleasure of reviewing Mallory’s Marv...
    
    

      
21
      
['category-autism-and-food']
      
https://www.autismparentingmagazine.com/sugary...
      
https://www.autismparentingmagazine.com/
      
Sivakumari asks: My daughter loves to eat ice-...
      
Q and A Do sugary foods make children with aut...
      
NaN
      
AutismParentingMagazine
      
[q, and, a, do, sugary, foods, make, children,...
      
Sivakumari asks: My daughter loves to eat ice-...
    
    

      
22
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/social...
      
https://www.autismparentingmagazine.com/
      
Children with autism spectrum disorder (ASD) c...
      
Social Stories for Autistic Children
      
NaN
      
AutismParentingMagazine
      
[social, stories, for, autistic, children, chi...
      
Children with autism spectrum disorder (ASD) c...
    
    

      
23
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/ways-h...
      
https://www.autismparentingmagazine.com/
      
How does your child react in the water—with pl...
      
Creative Ways to Help a Child with Autism Enjo...
      
NaN
      
AutismParentingMagazine
      
[creative, ways, to, help, a, child, with, aut...
      
How does your child react in the water—with pl...
    
    

      
24
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/autism...
      
https://www.autismparentingmagazine.com/
      
Over the years, lack of eye contact from a per...
      
Autism and Anxiety Disorder – The Double Diagn...
      
NaN
      
AutismParentingMagazine
      
[autism, and, anxiety, disorder, the, double, ...
      
Over the years, lack of eye contact from a per...
    
    

      
25
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/mental...
      
https://www.autismparentingmagazine.com/
      
If your child has autism spectrum disorder (AS...
      
Mental Health and Anxiety Treatment with Autis...
      
NaN
      
AutismParentingMagazine
      
[mental, health, and, anxiety, treatment, with...
      
If your child has autism spectrum disorder (AS...
    
    

      
26
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/help-a...
      
https://www.autismparentingmagazine.com/
      
Lately, the news has been immersed in images o...
      
Helping People With Autism Communicate in a Di...
      
NaN
      
AutismParentingMagazine
      
[help, people, with, autism, communicate, in, ...
      
Lately, the news has been immersed in images o...
    
    

      
27
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/raisin...
      
https://www.autismparentingmagazine.com/
      
When we brought my seven-year-old stepdaughter...
      
I See You: Raising Neurotypical and Autistic S...
      
NaN
      
AutismParentingMagazine
      
[i, see, you, raise, neurotypical, and, autist...
      
When we brought my seven-year-old stepdaughter...
    
    

      
28
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/servic...
      
https://www.autismparentingmagazine.com/
      
Families navigating through the world of autis...
      
The Awesome Rewards of a School Service Dog fo...
      
NaN
      
AutismParentingMagazine
      
[the, awesome, reward, of, a, school, service,...
      
Families navigating through the world of autis...
    
    

      
29
      
['category-autism-articles']
      
https://www.autismparentingmagazine.com/signs-...
      
https://www.autismparentingmagazine.com/
      
Early signs of autism spectrum disorder (ASD) ...
      
Signs of Autism in Infants and Children
      
NaN
      
AutismParentingMagazine
      
[sign, of, autism, in, infants, and, children,...
      
Early signs of autism spectrum disorder (ASD) ...
    
    

      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
      
...
    
    

      
270
      
forums
      
http://ehealthforum.com/health/is-it-possible-...
      
http://ehealthforum.com
      
While yes ASD is significantly related to gene...
      
is it possible that my kids got autism from me?
      
630934.0
      
eHealthForum
      
[be, it, possible, that, my, kid, get, autism,...
      
While yes ASD is significantly related to gene...
    
    

      
271
      
forums
      
http://ehealthforum.com/health/is-it-possible-...
      
http://ehealthforum.com
      
Regarding your dilemma about yourself, if you ...
      
is it possible that my kids got autism from me?
      
675981.0
      
eHealthForum
      
[be, it, possible, that, my, kid, get, autism,...
      
Regarding your dilemma about yourself, if you ...
    
    

      
272
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
i am 15 years old and was diagnosed with autis...
      
can autism be grown out of ?
      
578954.0
      
eHealthForum
      
[can, autism, be, grow, out, of, i, be, 15, ye...
      
i am 15 years old and was diagnosed with autis...
    
    

      
273
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
My 10 year old son has autism and I have studi...
      
can autism be grown out of ?
      
579216.0
      
eHealthForum
      
[can, autism, be, grow, out, of, my, 10, year,...
      
My 10 year old son has autism and I have studi...
    
    

      
274
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
I was diagnosed with Asperger's syndrome as a ...
      
can autism be grown out of ?
      
605369.0
      
eHealthForum
      
[can, autism, be, grow, out, of, i, be, diagno...
      
I was diagnosed with Asperger's syndrome as a ...
    
    

      
275
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
My name is Joseph. I have a daughter diagnosed...
      
can autism be grown out of ?
      
605429.0
      
eHealthForum
      
[can, autism, be, grow, out, of, my, name, be,...
      
My name is Joseph. I have a daughter diagnosed...
    
    

      
276
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
One can not grow out of autism, but coping str...
      
can autism be grown out of ?
      
313175.0
      
eHealthForum
      
[can, autism, be, grow, out, of, one, can, not...
      
One can not grow out of autism, but coping str...
    
    

      
277
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
There's evidence suggesting that gluten sensit...
      
can autism be grown out of ?
      
607499.0
      
eHealthForum
      
[can, autism, be, grow, out, of, there, s, evi...
      
There's evidence suggesting that gluten sensit...
    
    

      
278
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
Totally overcoming is not possible. But skills...
      
can autism be grown out of ?
      
621617.0
      
eHealthForum
      
[can, autism, be, grow, out, of, totally, over...
      
Totally overcoming is not possible. But skills...
    
    

      
279
      
forums
      
http://ehealthforum.com/health/can-autism-be-g...
      
http://ehealthforum.com
      
A big question from parents of kids with ADHD ...
      
can autism be grown out of ?
      
607499.0
      
eHealthForum
      
[can, autism, be, grow, out, of, a, big, quest...
      
A big question from parents of kids with ADHD ...
    
    

      
280
      
forums
      
http://ehealthforum.com/health/how-autism-affe...
      
http://ehealthforum.com
      
want to know how the Autism disease affects th...
      
How Autism Affects the Growth of the Child
      
709539.0
      
eHealthForum
      
[how, autism, affect, the, growth, of, the, ch...
      
want to know how the Autism disease affects th...
    
    

      
281
      
forums
      
http://ehealthforum.com/health/how-autism-affe...
      
http://ehealthforum.com
      
Children with autism spectrum disorder (ASD) d...
      
How Autism Affects the Growth of the Child
      
724934.0
      
eHealthForum
      
[how, autism, affect, the, growth, of, the, ch...
      
Children with autism spectrum disorder (ASD) d...
    
    

      
282
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
im 17 and last yera i found out i have autism ...
      
Can You Cure Autsim (Page 1)
      
92304.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, im, 17, and,...
      
im 17 and last yera i found out i have autism ...
    
    

      
283
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
unfortunatly to date there is no cure for auti...
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, unfortunatly...
      
unfortunatly to date there is no cure for auti...
    
    

      
284
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
well i find it really hard 2 talk 2 other peop...
      
Can You Cure Autsim (Page 1)
      
92304.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, well, i, fin...
      
well i find it really hard 2 talk 2 other peop...
    
    

      
285
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
I dont have autism but i have a very hard time...
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, i, dont, hav...
      
I dont have autism but i have a very hard time...
    
    

      
286
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
We never want to let the years pass by in weak...
      
Can You Cure Autsim (Page 1)
      
55269.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, we, never, w...
      
We never want to let the years pass by in weak...
    
    

      
287
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
Hey! You can get help. You should see a psycho...
      
Can You Cure Autsim (Page 1)
      
91919.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, hey, you, ca...
      
Hey! You can get help. You should see a psycho...
    
    

      
288
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
I was a complete skeptic regarding alternative...
      
Can You Cure Autsim (Page 1)
      
82225.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, i, be, a, co...
      
I was a complete skeptic regarding alternative...
    
    

      
289
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
Been using it myself for general health. The b...
      
Can You Cure Autsim (Page 1)
      
55269.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, be, use, it,...
      
Been using it myself for general health. The b...
    
    

      
290
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
anything with omega3 in it is good for the bra...
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, anything, wi...
      
anything with omega3 in it is good for the bra...
    
    

      
291
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
more than likely---ive seen some adults use it...
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, more, than, ...
      
more than likely---ive seen some adults use it...
    
    

      
292
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
ok thank you i will try it
      
Can You Cure Autsim (Page 1)
      
92304.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, ok, thank, y...
      
ok thank you i will try it
    
    

      
293
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
let us know how it works:)
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, let, us, kno...
      
let us know how it works:)
    
    

      
294
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
I did the same thing with my son who is on the...
      
Can You Cure Autsim (Page 1)
      
116173.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, i, do, the, ...
      
I did the same thing with my son who is on the...
    
    

      
295
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
yeah and remember that everyone is different a...
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, yeah, and, r...
      
yeah and remember that everyone is different a...
    
    

      
296
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
ok yeap thanks
      
Can You Cure Autsim (Page 1)
      
92304.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, ok, yeap, th...
      
ok yeap thanks
    
    

      
297
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
Personally, I has a similar problem myself wit...
      
Can You Cure Autsim (Page 1)
      
123989.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, personally, ...
      
Personally, I has a similar problem myself wit...
    
    

      
298
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
Right  Thank god for caller ID.
      
Can You Cure Autsim (Page 1)
      
91919.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, right, thank...
      
Right  Thank god for caller ID.
    
    

      
299
      
forums
      
http://ehealthforum.com/health/topic116210.html
      
http://ehealthforum.com
      
oh i agree completely with that. id bee doomed...
      
Can You Cure Autsim (Page 1)
      
71395.0
      
eHealthForum
      
[can, you, cure, autsim, page, 1, oh, i, agree...
      
oh i agree completely with that. id bee doomed...
    
  

300 rows × 9 columns

In [24]:

    

# Tokenize data
import nltk
tokenizer = nltk.RegexpTokenizer(r'\w+')

# Get list of tokens from text in first article:
text = df['text'][0].lower()
# Add also title:
title = df['title'][0].lower()
ttext = tokenizer.tokenize(title+" "+text)


    

In [25]:

    

# Get a column with list of tokens:

# 1) convert to lower case 
# 2) get tokens
# 2) save data in a new column (tokens)
#df['tokens'] = df['text'].map(lambda x: tokenizer.tokenize(x.lower()))

# Join title and text into one column
df2=df[['title','text']].apply(lambda x: ','.join(x.astype(str)),axis=1)
df['tokens'] = df2.map(lambda x: tokenizer.tokenize(x.lower()))
del df2


    

In [26]:

    

# Short version of text
df['text_short']=df['text'].apply(lambda x: x[:200]+" (...)" if (len(x) > 200) else x)


    

In [27]:

    

# Lematize

from nltk.stem.wordnet import WordNetLemmatizer

for ii in df.index:
    tokens = df.loc[ii,'tokens']
    lemas=[]
    for word in tokens:
        ww=WordNetLemmatizer().lemmatize(word,'v')
        lemas.append(ww)
    df.set_value(ii,'tokens',lemas)

#


    

In [28]:

    

# Save dataframe with tokens into files
df.to_csv(output_fname)


    

In [14]:

    

# Get similarity matrices
documents = df['tokens'].values
print(documents[:3])
corpus,dictionary = make_corpus(documents)

#Save corpus into file
import pickle
pickle.dump(dictionary,open("dictionary.save","wb"))
pickle.dump(corpus,open("corpus.save", "wb"))

tfidf = models.TfidfModel(corpus)
tfidf.save('tfidf.save')

lsi_matsim = make_lsi_similarity_matrix(tfidf[corpus], dictionary)
lda_matsim = make_lda_similarity_matrix(corpus, dictionary)

# The models are saved into files in the above routines
# Save similarity matrices too:
pickle.dump(lsi_matsim,open("lsi-matsim.save","wb"))
pickle.dump(lda_matsim,open("lda-matsim.save","wb"))


    

    




[ ['autism', 'head', 'bang', 'and', 'other', 'self', 'harm', 'behavior', 'for', 'children', 'with', 'autism', 'spectrum', 'disorder', 'asd', 'head', 'bang', 'be', 'a', 'common', 'way', 'to', 'self', 'soothe', 'and', 'communicate', 'need', 'both', 'neurotypical', 'and', 'autistic', 'baby', 'and', 'toddlers', 'seek', 'to', 'recreate', 'the', 'rhythm', 'that', 'stimulate', 'their', 'vestibular', 'system', 'while', 'in', 'utero', 'other', 'rhythmic', 'habit', 'that', 'fuel', 'a', 'child', 's', 'kinesthetic', 'drive', 'include', 'head', 'roll', 'body', 'rock', 'bite', 'and', 'thumb']
 ['high', 'quality', 'aba', 'treatment', 'what', 'every', 'parent', 'need', 'to', 'know', 'dr', 'stephen', 'shore', 'once', 'say', 'if', 'you', 've', 'meet', 'one', 'person', 'with', 'autism', 'you', 've', 'meet', 'one', 'person', 'with', 'autism', 'as', 'many', 'of', 'you', 'know', 'the', 'centre', 'for', 'disease', 'control', 'and', 'prevention', 'report', 'that', 'there', 'be', 'now', '1', 'in', '68', 'children', 'live', 'with', 'autism', 'in', 'the', 'unite', 'state', 'what', 'this', 'mean', 'for', 'each', 'child', 'and', 'his', 'her', 'family']
 ['help', 'i', 'don', 't', 'know', 'how', 'to', 'choose', 'an', 'apply', 'behavior', 'analysis', 'provider', 'help', 'i', 'be', 'go', 'to', 'be', 'start', 'apply', 'behavior', 'analysis', 'for', 'my', 'son', 'who', 'have', 'autism', 'can', 'you', 'tell', 'me', 'things', 'to', 'ask', 'and', 'what', 'to', 'look', 'for', 'in', 'an', 'aba', 'provider', 'jane', 'i', 'be', 'so', 'glad', 'you', 'ask', 'this', 'within', 'the', 'autism', 'community', 'there', 'be', 'mix', 'feel', 'about', 'apply', 'behavior', 'analysis', 'aba', 'but']]

In [ ]: