

In [1]:

    
### Get comments by using Facebook API ###
import facebook
import requests

### Cleaning the comments using nltk ###
import re
import nltk
#nltk.download('stopwords') # Uncomment to download stopwords
from nltk.corpus import stopwords
from nltk.stem.snowball import SpanishStemmer

### Create Bag of Words model ###
from sklearn.feature_extraction.text import CountVectorizer

### Create Bag of Words model ###
from sklearn.feature_extraction.text import CountVectorizer

### Web Scrapping ###
from bs4 import BeautifulSoup

### Saving data ###
import os

def get_id_link(id='prensalibregt/posts?'):
    access_token = 'EAACEdEose0cBAGX0IhGJg7mNAEoBWZCZBs1n6WrTcuoMDzTpihe1iLkGSPoa4MFGodJS9AZAi4ffPBVrIq23RLEJOg868udul63CAEoPsrZABfhOZB1fSRkA7d7s3ZBxrvkPEn3h9BPGB3wU1Gc3vOTEK1GGZBeZARb0AqPsjDWScrfCCz54dZAV7q4g4Fh2UX3aNTtHNUZCtZAugZDZD'
    graph = facebook.GraphAPI(access_token)
    post = graph.get_object(id=id, fields='link')
    
    id_news = []
    link_news = []
    
    for posts in post['data']:
        id_news.append(posts['id'])
        link_news.append(posts['link'])
    return [id_news,  link_news] 


def get_facebook_comments(news_id):
    comments = get_comments(news_id)
    corpus = clean_comments(comments)
    words = bag_of_words(corpus, max_features = 10)
    return [comments, corpus, words]


def get_comments(news_id):
    # Facebook Access Token: https://developers.facebook.com/tools/explorer/
    access_token = 'EAACEdEose0cBAGX0IhGJg7mNAEoBWZCZBs1n6WrTcuoMDzTpihe1iLkGSPoa4MFGodJS9AZAi4ffPBVrIq23RLEJOg868udul63CAEoPsrZABfhOZB1fSRkA7d7s3ZBxrvkPEn3h9BPGB3wU1Gc3vOTEK1GGZBeZARb0AqPsjDWScrfCCz54dZAV7q4g4Fh2UX3aNTtHNUZCtZAugZDZD'

    user = '/me'
    graph = facebook.GraphAPI(access_token)
    profile = graph.get_object(user)

    # Testing with Prensa Libre's posts
    posts = graph.get_connections(id =news_id, connection_name='comments')
 
    comments = []
    while True:
        try:
            for post in posts['data']:
                comments.append(post['message'])
            posts = requests.get(posts['paging']['next']).json()
        except KeyError:
            break
    
    return comments


def clean_comments(comments):
    corpus = []
    for i in range(0,len(comments)):
        review = re.sub('[^a-zA-Z]', ' ', comments[i])
        review = review.lower()
        review = review.split()
        stemmer = SpanishStemmer()
        review = [stemmer.stem(word) for word in review if not word in set(stopwords.words('spanish'))]
        review = ' '.join(review)
        corpus.append(review)
        
    return corpus


def bag_of_words(corpus, max_features = 5):
    cv = CountVectorizer(max_features = max_features)
    X = cv.fit_transform(corpus).toarray()
    return cv.get_feature_names() 


def getNoticia(url):
    # Capturamos la url ingresada en la variable "url"
    url = url
    r  = requests.get(url)
    data = r.text
    
    # Creamos el objeto soup y le pasamos lo capturado con request
    soup = BeautifulSoup(data, "html.parser")

    #aqui se toman los articulos extra de prensalibre para removerlos al final y dejar solo la noticia de interes
    articles = soup.find_all('article', {'class' : 'story related gi'})
    numArticulos = 0
    for article in articles:
        numArticulos+=1

    # Find all of the text between paragraph tags and strip out the html
    parrafos = soup.find_all('p', {'class' : ''})
    listaParrafos = []
    for parrafo in parrafos:
        listaParrafos.append(parrafo.getText())


    #obtener solo los parrafos de contenido relevante (no noticias adicionales, publicidad, etc...)
    limiteSinArticulos = len(listaParrafos)- numArticulos#len(listaArticulos)
    cadena = ""
    for i in range(0,limiteSinArticulos):
        cadena += listaParrafos[i]
    return cadena

def get_News(url):
    listNews = []
    news = getNoticia(url)
    listNews.append(news)
    corpus = clean_comments(listNews)
    words = bag_of_words(corpus, max_features = 10)
    return [listNews, corpus, words]


### Auxiliary function for making array of words ###
def palabraComentario (lista1, lista2):
    listaT = []
    for p in lista1:
        for c in lista2:
            listaT.append([p,c])
    return listaT


### Auxiliary functions for plotting ###
def dispersion_plot(corpus, words):
    tokens = []
    for word in corpus:
        tokens = tokens + nltk.word_tokenize(word)
    text = nltk.Text(tokens)
    text.dispersion_plot(words)
    return

def frequency_plot(corpus, words):
    tokens = []
    for word in corpus:
        tokens = tokens + nltk.word_tokenize(word)
    text = nltk.Text(tokens)
    fd = nltk.FreqDist(text)
    fd.plot(50,cumulative=False)
    return

### Auxiliary function for saving data ###
def save_data(file, data):
    file = open(file, "w")
    for e in data:
        file.write(e[0] + "," + e[1] + os.linesep)
    file.close()