如何爬取Facebook粉絲頁資料 (comments) ？

In [1]:

    

# 載入python 套件
import requests
import datetime
import time
import pandas as pd


    

In [2]:

    

# 粉絲頁的id
page_id = "appledaily.tw"

# 應用程式的帳號,密碼
app_id = ""
app_secret = ""

# 上一篇爬取的post的csv檔案
post_path = 'post/'+page_id+'_post.csv'

access_token = app_id + "|" + app_secret


    

In [3]:

    

# 判斷response有無正常 正常 200，若無隔五秒鐘之後再試
def request_until_succeed(url):
    success = False
    while success is False:
        try: 
            req = requests.get(url)
            if req.status_code == 200:
                success = True
            if req.status_code == 400:
                return None
        except Exception as e:
            print(e)
            time.sleep(5)
            print("Error for URL %s: %s" % (url, datetime.datetime.now()))
            print("Retrying.")
    return req


    

In [4]:

    

def getFacebookCommentFeedData(status_id, access_token, num_comments):
    
    base = "https://graph.facebook.com/v2.6"
    node = "/%s/comments" % status_id
    fields = "?fields=id,message,like_count,created_time,comments,from,attachment"
    parameters = "&order=chronological&limit=%s&access_token=%s" % \
            (num_comments, access_token)
    url = base + node + fields + parameters

    # 取得data
    data = request_until_succeed(url)
    if data is None:
        return None
    else:
        return data.json()


    

In [5]:

    

def processFacebookComment(comment, status_id, parent_id = ''):
    
    # 確認資料欄位是否有值,並做處理
    comment_id = comment['id']
    comment_author = comment['from']['name']
    if 'message' not in comment:
        comment_message = ''
    else:
        comment_message = comment['message']
    if 'like_count' not in comment:
        comment_likes = 0 
    else:
        comment_likes = comment['like_count']
    
    if 'attachment' in comment:
        attach_tag = "[[%s]]" % comment['attachment']['type'].upper()
        if comment_message is '':
            comment_message = attach_tag
        else:
            comment_message = (comment_message+ " " +attach_tag)

    comment_published = datetime.datetime.strptime(comment['created_time'],'%Y-%m-%dT%H:%M:%S+0000')
     # 根據所在時區 TW +8
    comment_published = comment_published + datetime.timedelta(hours=8)
    comment_published = comment_published.strftime('%Y-%m-%d %H:%M:%S')
    
    # 回傳tuple形式的資料
    return (comment_id, status_id, parent_id, comment_message, comment_author,
            comment_published, comment_likes)


    

In [6]:

    

def scrapeFacebookPageFeedComments(page_id, access_token, post_path):
    
    # all_statuses 用來儲存的list,先放入欄位名稱
    all_comments = [("comment_id", "status_id", "parent_id", "comment_message",
        "comment_author", "comment_published", "comment_likes")]

    num_processed = 0   # 計算處理多少post
    scrape_starttime = datetime.datetime.now()

    print("Scraping %s Comments From Posts: %s\n" % (page_id, scrape_starttime))
    
    post_df = pd.read_csv(post_path)

    for status_id in post_df['status_id']:
        has_next_page = True

        comments = getFacebookCommentFeedData(status_id, access_token, 100)

        while has_next_page and comments is not None:
            for comment in comments['data']:
                
                all_comments.append(processFacebookComment(comment, status_id))
                if 'comments' in comment:
                    has_next_subpage = True

                    subcomments = getFacebookCommentFeedData(comment['id'], access_token, 100)

                    while has_next_subpage:
                        for subcomment in subcomments['data']:
                            all_comments.append(processFacebookComment(
                                    subcomment,
                                    status_id,
                                    comment['id']))

                            num_processed += 1
                            if num_processed % 1000 == 0:
                                print("%s Comments Processed: %s" %
                                      (num_processed,
                                       datetime.datetime.now()))

                        if 'paging' in subcomments:
                            if 'next' in subcomments['paging']:
                                data =  request_until_succeed(subcomments['paging']['next'])
                                if data != None:
                                    subcomments = data.json()
                                else:
                                    subcomments = None
                            else:
                                has_next_subpage = False
                        else:
                            has_next_subpage = False
                            
                num_processed += 1
                if num_processed % 1000 == 0:
                    print("%s Comments Processed: %s" %
                          (num_processed, datetime.datetime.now()))

            if 'paging' in comments:
                if 'next' in comments['paging']:
                    data =  request_until_succeed(comments['paging']['next'])
                    if data != None:
                        comments = data.json()
                    else:
                        comments = None
                else:
                    has_next_page = False
            else:
                has_next_page = False
                
    print("\nDone!\n%s Comments Processed in %s" %
          (num_processed, datetime.datetime.now() - scrape_starttime))
    return all_comments


    

In [8]:

    

all_comments = scrapeFacebookPageFeedComments(page_id, access_token, post_path)
df = pd.DataFrame(all_comments[1:], columns=all_comments[0])


    

In [9]:

    

path = 'comment/'+page_id+'_comment.csv'
df.to_csv(path,index=False,encoding='utf8')


    

In [ ]: