In [1]:
import requests  # to make GET request
from bs4 import BeautifulSoup  # to parse the HTML response
import time  # to pause between calls
import pandas as pd  # to see CSV
import os

In [2]:
def parse_page(response):
    """This parses pages from the Autism Parent Magazine web site using beautifulsoup"""
    global df
    global source
    # parse the response into an HTML tree soup object
    # read the content of the server’s response as a string
    page_source = response.text
    soup = BeautifulSoup(page_source, 'html5lib')
    # take a look
    #print(soup.prettify()[:1000])

    # Find all "<div class="entry-row" appearances
    links=soup.select("div.entry-row")
    for link in links:
        article=link.select("article")
        # Get category (for validation)
        for element in article[0]['class']:
            if ( 'category' in element ):
                category=element
                break
                
        # Get text
        #article[0].find_all('div')[1].get_text()
        text=article[0].find('div', class_='entry-summary entry-content').get_text()
    
        # Get href. for further reading
        href=article[0].find('a')['href']
    
        # Get title
        title=article[0].find('a')['title']
        
        newrow={"title":title,"source":source,"category":category,"text":text,"href":href}

        # Add entry into database
        df.loc[len(df.values)]=newrow

In [3]:
os.chdir("../data")

# Create an empty database to keep the history of purchases
columns=["title","source","category","text","href"]
df=pd.DataFrame(columns=columns)

In [4]:
source="https://www.autismparentingmagazine.com/"
    
# Get list of pages to parse:
f = open("AutismParentMagazine-urls.txt","r")
urls =  f.readlines()
f.close()

for ii in range(len(urls)):
    url = urls[ii]
    urls[ii]=url.strip()

In [5]:
#loop over urls:

for url in urls:
    # make a GET request
    response = requests.get(url)

    parse_page(response)
    time.sleep(5)

In [7]:
# Convert to file
df.to_csv("AutismParentMagazine-posts.csv")

In [8]:
df


Out[8]:
title source category text href
0 Autism, Head Banging and other Self Harming Be... https://www.autismparentingmagazine.com/ category-applied-behavior-analysis-aba For children with autism spectrum disorder (AS... https://www.autismparentingmagazine.com/autism...
1 High Quality ABA Treatment:  What Every Parent... https://www.autismparentingmagazine.com/ category-applied-behavior-analysis-aba Dr. Stephen Shore once said “If you’ve met one... https://www.autismparentingmagazine.com/high-q...
2 Help: I Don’t Know How to Choose an Applied Be... https://www.autismparentingmagazine.com/ category-applied-behavior-analysis-aba Help! I am going to be starting Applied Behav... https://www.autismparentingmagazine.com/choosi...
3 HELP: My Autistic Child is Absolutely Terrifie... https://www.autismparentingmagazine.com/ category-applied-behavior-analysis-aba How do you handle high anxiety of a child on t... https://www.autismparentingmagazine.com/help-a...
4 HELP: I Need Communication Advice for Autistic... https://www.autismparentingmagazine.com/ category-applied-behavior-analysis-aba A grandfather from Singapore asks… My eldest g... https://www.autismparentingmagazine.com/help-i...
5 Help: My Aspie Lacks Motivation https://www.autismparentingmagazine.com/ category-applied-behavior-analysis-aba Hi, I am the parent of a 13-year-old boy with ... https://www.autismparentingmagazine.com/help-m...
6 Exciting Campaign Launched to Tackle the Autis... https://www.autismparentingmagazine.com/ category-autism-advocacy It will be some time before we see a positive ... https://www.autismparentingmagazine.com/autism...
7 How One Restaurateur Makes Eating Out Autism-F... https://www.autismparentingmagazine.com/ category-autism-advocacy Restaurant owner Andrew Iredale could hardly b... https://www.autismparentingmagazine.com/restau...
8 Lloyd Claycomb’s Views on Autism https://www.autismparentingmagazine.com/ category-autism-advocacy Autism can be one of the most difficult diagno... https://www.autismparentingmagazine.com/lloyd-...
9 Erasing the Prejudice – Are People on the Spec... https://www.autismparentingmagazine.com/ category-autism-advocacy An Interview with Dr. Tony Attwood, PhD, AS/AS... https://www.autismparentingmagazine.com/erasin...
10 The Never Ending Battle for Autism Rights – A ... https://www.autismparentingmagazine.com/ category-autism-advocacy A civil rights lawyer and autism mom talks abo... https://www.autismparentingmagazine.com/the-ne...
11 How One Special Bear Changed the Way People Se... https://www.autismparentingmagazine.com/ category-autism-advocacy In two short years, Tikko has captured the hea... https://www.autismparentingmagazine.com/how-on...
12 Winning the Fight for Autism https://www.autismparentingmagazine.com/ category-autism-advocacy Every time the professional fighters meet thei... https://www.autismparentingmagazine.com/winnin...
13 The Right for Autism Services for U.S. Militar... https://www.autismparentingmagazine.com/ category-autism-advocacy Autism Services for Military Families by Lesli... https://www.autismparentingmagazine.com/autism...
14 Bridge the Gap Gala Raises over $100,000 https://www.autismparentingmagazine.com/ category-autism-advocacy Bridge the Gap Gala Raises over $100,000 by Le... https://www.autismparentingmagazine.com/bridge...
15 Exposing Autism Speaks – Podcast 4 https://www.autismparentingmagazine.com/ category-autism-advocacy We started this magazine to help parents, so... https://www.autismparentingmagazine.com/exposi...
16 Top Ways a Gluten Free Diet  Can Help Kids wit... https://www.autismparentingmagazine.com/ category-autism-and-diet Recent clinical trials looking at gluten-free ... https://www.autismparentingmagazine.com/gluten...
17 Asperger’s Syndrome and the Power of a Good Diet https://www.autismparentingmagazine.com/ category-autism-and-diet I find the suggestion that there needs to be a... https://www.autismparentingmagazine.com/asperg...
18 Six Steps to a Fun, Friendly, Family Fitness P... https://www.autismparentingmagazine.com/ category-autism-and-diet “Wow, how did you get him/her to do that?,” pa... https://www.autismparentingmagazine.com/six-st...
19 Q&A Section: How does Gluten free casein free ... https://www.autismparentingmagazine.com/ category-autism-and-diet This is the question of the month as featured ... https://www.autismparentingmagazine.com/qa-sec...
20 Mallory’s Marvelous Muffins https://www.autismparentingmagazine.com/ category-autism-and-food I had the pleasure of reviewing Mallory’s Marv... https://www.autismparentingmagazine.com/mallor...
21 Q and A Do sugary foods make children with aut... https://www.autismparentingmagazine.com/ category-autism-and-food Sivakumari asks: My daughter loves to eat ice-... https://www.autismparentingmagazine.com/sugary...
22 Social Stories for Autistic Children https://www.autismparentingmagazine.com/ category-autism-articles Children with autism spectrum disorder (ASD) c... https://www.autismparentingmagazine.com/social...
23 Creative Ways to Help a Child with Autism Enjo... https://www.autismparentingmagazine.com/ category-autism-articles How does your child react in the water—with pl... https://www.autismparentingmagazine.com/ways-h...
24 Autism and Anxiety Disorder – The Double Diagn... https://www.autismparentingmagazine.com/ category-autism-articles Over the years, lack of eye contact from a per... https://www.autismparentingmagazine.com/autism...
25 Mental Health and Anxiety Treatment with Autis... https://www.autismparentingmagazine.com/ category-autism-articles If your child has autism spectrum disorder (AS... https://www.autismparentingmagazine.com/mental...
26 Helping People With Autism Communicate in a Di... https://www.autismparentingmagazine.com/ category-autism-articles Lately, the news has been immersed in images o... https://www.autismparentingmagazine.com/help-a...
27 I See You: Raising Neurotypical and Autistic S... https://www.autismparentingmagazine.com/ category-autism-articles When we brought my seven-year-old stepdaughter... https://www.autismparentingmagazine.com/raisin...
28 The Awesome Rewards of a School Service Dog fo... https://www.autismparentingmagazine.com/ category-autism-articles Families navigating through the world of autis... https://www.autismparentingmagazine.com/servic...
29 Signs of Autism in Infants and Children https://www.autismparentingmagazine.com/ category-autism-articles Early signs of autism spectrum disorder (ASD) ... https://www.autismparentingmagazine.com/signs-...
... ... ... ... ... ...
204 KinderKord: A Safe Way To Stay Connected https://www.autismparentingmagazine.com/ category-autism-articles KinderKord by Joan Lunden   KinderKord is a ... https://www.autismparentingmagazine.com/kinder...
205 My Signals: Autistic Reflections of a Man Who ... https://www.autismparentingmagazine.com/ category-personal-narrative My Signals: Autistic Reflections by Patrick Ja... https://www.autismparentingmagazine.com/signal...
206 Go-Go Blanket: The perfect travel blanket for ... https://www.autismparentingmagazine.com/ category-autism-news Review of the Go-Go blanket by Go-Go Blanket... https://www.autismparentingmagazine.com/go-go-...
207 Black and White: A Colorful Look at the Spectrum https://www.autismparentingmagazine.com/ category-autism-books Black and White: A Colorful Look at the Spect... https://www.autismparentingmagazine.com/black-...
208 Sparrow Migrations: A Novel https://www.autismparentingmagazine.com/ category-autism-books Sparrow Migrations: A Novel by Cari Noga   Neu... https://www.autismparentingmagazine.com/sparro...
209 Creative Ways to Help a Child with Autism Enjo... https://www.autismparentingmagazine.com/ category-autism-articles How does your child react in the water—with pl... https://www.autismparentingmagazine.com/ways-h...
210 Easy Ways to Create A Sensory Space for Kids w... https://www.autismparentingmagazine.com/ category-autism-articles Children with autism spectrum disorder (ASD) m... https://www.autismparentingmagazine.com/creati...
211 Simple Ways to Make Bath Time a Splash with Se... https://www.autismparentingmagazine.com/ category-autism-articles Like so many parents-to-be, when I was pregnan... https://www.autismparentingmagazine.com/bathin...
212 Simple Ways to Help Kids with Autism Develop H... https://www.autismparentingmagazine.com/ category-autism-articles I learned almost 30 years ago that my extremel... https://www.autismparentingmagazine.com/simple...
213 Simple Ways Sensory Based Intervention Can Cha... https://www.autismparentingmagazine.com/ category-autism-articles One in six children experiences sensory diffic... https://www.autismparentingmagazine.com/sensor...
214 Amazing New Fidget Pillow Calms Busy Hands and... https://www.autismparentingmagazine.com/ category-autism-articles My desire to help my child be successful in sc... https://www.autismparentingmagazine.com/new-fi...
215 5 Top Reasons Qigong May be Right for Your Spe... https://www.autismparentingmagazine.com/ category-autism-articles By Sazini Nzula, PhD, Certified QST Therapist ... https://www.autismparentingmagazine.com/qigong...
216 Brain Balance Takes a Look at the Challenge of... https://www.autismparentingmagazine.com/ category-autism-articles “I can’t wear those jeans!” “No, I won’t eat m... https://www.autismparentingmagazine.com/brain-...
217 Putting the Squeeze On Sensory Processing Diso... https://www.autismparentingmagazine.com/ category-autism-articles Industrial Designer Lisa Fraser never imagined... https://www.autismparentingmagazine.com/puttin...
218 Sensory Processing Difficulties – Dressing For... https://www.autismparentingmagazine.com/ category-autism-articles What is Sensory Processing disorder? Many chil... https://www.autismparentingmagazine.com/sensor...
219 Simple Ways to Help Your Child with ASD Sleep ... https://www.autismparentingmagazine.com/ category-autism-articles Insomnia (sleeplessness) is quite common among... https://www.autismparentingmagazine.com/autism...
220 Sure Fire Ways to Help Your ASD Child Go To Bed https://www.autismparentingmagazine.com/ category-autism-articles In order to have a good day, your child must s... https://www.autismparentingmagazine.com/sure-f...
221 The Calming Effects of Essential Oils https://www.autismparentingmagazine.com/ category-autism-articles More and more we are hearing about essential o... https://www.autismparentingmagazine.com/the-ca...
222 Helping Your Child with Autism Sleep: An Overv... https://www.autismparentingmagazine.com/ category-autism-articles Sleep problems are common amongst children. Fo... https://www.autismparentingmagazine.com/helpin...
223 Simple Ways to Make Social Skill Connections U... https://www.autismparentingmagazine.com/ category-autism-articles When you first hear that a computer game or a ... https://www.autismparentingmagazine.com/social...
224 Special Interactive Program for Kids Encourage... https://www.autismparentingmagazine.com/ category-autism-articles “You’re having twins!” Can you imagine the exc... https://www.autismparentingmagazine.com/specia...
225 4 Simple Steps to Help Your Child PREP for Soc... https://www.autismparentingmagazine.com/ category-autism-articles We have been working with families of children... https://www.autismparentingmagazine.com/prep-f...
226 5 Easy Ways To Help Improve Executive Functioning https://www.autismparentingmagazine.com/ category-autism-articles Rick walks into his classroom, and he is angry... https://www.autismparentingmagazine.com/ways-i...
227 Tips for Teaching the Most Important Social Sk... https://www.autismparentingmagazine.com/ category-autism-articles As a child, we had a player piano in my home. ... https://www.autismparentingmagazine.com/most-i...
228 5 Simple Ways to Help Your Child with ASD Cope... https://www.autismparentingmagazine.com/ category-autism-articles Welcome back to Social Skills Corner. For the ... https://www.autismparentingmagazine.com/cope-w...
229 Top 5 Social Skills Tips for Teens with Specia... https://www.autismparentingmagazine.com/ category-autism-articles Social skills are especially difficult for tee... https://www.autismparentingmagazine.com/social...
230 How to Help Your Child Develop Social Skills T... https://www.autismparentingmagazine.com/ category-autism-articles Socializing forms a big part of our everyday l... https://www.autismparentingmagazine.com/social...
231 5 Things You Need to Know About Social Skills https://www.autismparentingmagazine.com/ category-autism-articles Welcome to Social Skills Corner.  We have been... https://www.autismparentingmagazine.com/knowin...
232 10 Valuable Ways to Cope with Social Interaction https://www.autismparentingmagazine.com/ category-autism-articles Coping with Asperger’s syndrome can be a day-t... https://www.autismparentingmagazine.com/10-val...
233 Using Mindfulness to Conquer Parental Stress Now https://www.autismparentingmagazine.com/ category-autism-articles As the parent of a child with autism spectrum ... https://www.autismparentingmagazine.com/using-...

234 rows × 5 columns


In [ ]: