In [1]:
%run 'analyseArticle.ipynb'
%run 'examineArticle.ipynb'
In [2]:
import requests
from bs4 import BeautifulSoup
URLs = ["mothership.sg/category/news","mothership.sg/category/perspectives","mothership.sg/category/community","mothership.sg/category/almost-famous","mothership.sg/category/mps-in-the-house","mothership.sg/category/humour"]
for url in URLs:
sourceCode = requests.get("http://" + str(url))
soup = BeautifulSoup(sourceCode.content, "lxml")
for div in soup.find_all("div", class_="ind-article"):
for a in div.find_all("a"):
if "mothership.sg" in a.get("href"):
print(a.get("href"))
https://mothership.sg/2017/10/famous-hokkaido-stretchy-cheese-toast-brand-in-spore-to-seek-opportunities-to-open-here/
https://mothership.sg/2017/10/health-sciences-authority-rebuts-claim-its-officers-acted-like-thugs/
https://mothership.sg/2017/10/fake-news-groom-in-china-did-not-die-after-being-dropped-on-his-head-during-wedding-game/
https://mothership.sg/2017/10/sph-to-cut-230-jobs-130-of-these-will-be-retrenchments/
https://mothership.sg/2017/10/spore-self-heating-hotpot-distributor-blames-agency-for-not-declaring-meat-in-instant-hotpots/
https://mothership.sg/2017/10/worried-about-religious-divide-malaysias-monarchs-make-rare-collective-call-for-greater-tolerance-and-respect/
https://mothership.sg/2017/10/pm-lee-set-to-shake-us-president-donald-trumps-hand-for-second-time-on-oct-23/
https://mothership.sg/2017/10/cna-producer-juwon-park-repeatedly-discouraged-from-complaining-to-hr-about-sexism/
https://mothership.sg/2017/10/news-of-taiwanese-villagers-protesting-against-spore-is-6-months-old-not-new/
https://mothership.sg/2017/10/about-25-percent-of-singapores-students-have-been-bullied-in-school/
https://mothership.sg/2017/10/an-open-letter-from-a-former-struggling-student-in-spore/
https://mothership.sg/2017/10/a-formal-letter-to-cai-png-aunties-uncles-in-spore/
https://mothership.sg/2017/10/so-will-mrt-work-on-monday-sporeans-plan-ahead-without-credible-transport-alternatives/
https://mothership.sg/2017/10/chinese-sporean-woman-recognised-her-chinese-ness-after-strange-men-hit-on-her-in-europe/
https://mothership.sg/2017/10/pap-nervous-pm-esm-2-dpms-2-other-ministers-have-addressed-elected-presidency-anger/
https://mothership.sg/2017/10/bilahari-kausikan-asean-is-imperfect-but-its-incoherence-sometimes-is-not-a-bad-thing/
https://mothership.sg/2017/10/hazy-talk-we-measure-the-relative-psi-of-every-minister-who-spoke-on-counting-elected-presidency-terms/
https://mothership.sg/2017/10/isis-falsely-claimed-responsibility-for-las-vegas-shooting-the-media-had-to-report-on-it/
https://mothership.sg/2017/10/new-york-times-article-praises-spores-healthcare-system-to-high-heavens/
https://mothership.sg/2017/09/workers-partys-new-hq-on-outskirts-of-geylang-is-a-modest-reflection-of-where-they-are-now/
https://mothership.sg/2017/10/mph-has-stood-for-different-names-in-the-past-but-they-still-sell-books/
https://mothership.sg/2017/10/hdb-things-we-are-so-used-to-now-but-will-probably-not-miss-in-10-years-time/
https://mothership.sg/2017/10/straits-times-reports-vending-machines-selling-halal-certified-headscarves-everyone-confused/
https://mothership.sg/2017/10/this-is-the-story-of-a-whatsapp-bro-named-xiao-andy-who-broke-the-bro-code-for-100/
https://mothership.sg/2017/10/if-you-are-already-applying-for-a-credit-card-why-not-apply-it-here-and-get-150-grab-credits/
https://mothership.sg/2017/10/a-formal-letter-to-cai-png-aunties-uncles-in-spore/
https://mothership.sg/2017/10/yes-it-was-indeed-speaker-of-parl-tan-chuan-jin-eating-bak-kut-teh-with-wp-chief-low-thia-khiang/
https://mothership.sg/2017/10/sporeans-are-one-of-the-fastest-dumb-phone-texters-in-the-world/
https://mothership.sg/2017/10/about-25-percent-of-singapores-students-have-been-bullied-in-school/
https://mothership.sg/2017/10/if-you-forgot-to-take-your-s1190-from-woodlands-town-centre-posb-atm-a-good-samaritan-has-it/
https://mothership.sg/2017/10/the-sporean-whose-bright-academic-career-in-spore-ended-for-non-academic-reasons/
https://mothership.sg/2017/09/nac-deputy-ceo-congratulated-sonny-liew-actually-said-the-art-of-charlie-chan-hock-chye/
https://mothership.sg/2017/08/salleh-marican-feels-sporeans-will-be-angry-if-they-dont-get-a-chance-to-vote-for-their-president/
https://mothership.sg/2017/08/presidential-hopeful-farid-khan-doesnt-just-want-to-be-a-recipient-of-anti-papgovt-votes/
https://mothership.sg/2017/08/how-halimah-yacob-found-herself-in-the-running-to-be-spores-first-female-president/
https://mothership.sg/2017/07/spores-no-1-salesman-beh-swan-gin-is-a-trained-medical-doctor-who-cant-save-lives/
https://mothership.sg/2017/07/chan-chun-sing-doesnt-think-his-military-background-is-a-limitation-leadership-style-is-to-value-add/
https://mothership.sg/2017/07/internationally-famous-for-sporean-beatboxer-dharni-the-world-is-truly-his-oyster/
https://mothership.sg/2017/06/almost-famous-we-interview-ourselves-to-find-out-why-our-videos-suddenly-look-so-kilat/
https://mothership.sg/2017/06/almost-famous-spores-only-full-time-professional-clown-is-now-the-worlds-best/
https://mothership.sg/2017/10/about-25-percent-of-singapores-students-have-been-bullied-in-school/
https://mothership.sg/2017/10/reporters-without-borders-whack-spore-ministers-whack-reporters-without-borders-repeat/
https://mothership.sg/2017/10/job-seekers-can-look-forward-to-better-recruitment-standards-but-they-still-may-not-be-informed-of-rejections/
https://mothership.sg/2017/10/we-have-to-win-the-fight-for-hearts-and-minds-k-shanmugam-on-terrorism/
https://mothership.sg/2017/10/here-are-the-full-speeches-given-by-sylvia-lim-k-shanmugam-on-the-reserved-presidential-election/
https://mothership.sg/2017/10/deciding-who-is-the-first-elected-president-is-a-policy-decision-not-a-legal-matter-k-shanmugam/
https://mothership.sg/2017/10/new-anti-smoking-bill-means-you-have-to-be-21-before-you-can-smoke/
https://mothership.sg/2017/10/there-were-more-than-400-cases-of-domestic-violence-against-older-women-every-year-since-2014-confirms-desmond-lee/
https://mothership.sg/2017/10/third-time-lucky-for-sylvia-lim-as-her-adjournment-motion-on-the-reserved-presidency-was-picked-from-the-ballot/
https://mothership.sg/2017/10/buikt-panjang-peeps-rejoice-tender-for-lrt-overhaul-to-be-called-by-end-2017/
https://mothership.sg/2017/10/this-is-the-story-of-a-whatsapp-bro-named-xiao-andy-who-broke-the-bro-code-for-100/
https://mothership.sg/2017/10/woman-evades-paying-10-90-taxi-fare-for-a-week-says-she-is-busy-looking-for-money/
https://mothership.sg/2017/10/sporeans-are-one-of-the-fastest-dumb-phone-texters-in-the-world/
https://mothership.sg/2017/10/geyser-spotted-in-jurong-west-becomes-3rd-water-feature-to-spontaneously-appear-in-western-spore/
https://mothership.sg/2017/10/do-chio-bus-get-more-cai-png-ingredients-son-of-stall-owners-answers-this-and-more-in-qa/
https://mothership.sg/2017/10/ava-fines-importers-of-self-heating-instant-hotpots-after-straits-times-publicised-it/
https://mothership.sg/2017/10/mega-yacht-owned-by-microsoft-co-founder-paul-allen-spotted-in-singapore/
https://mothership.sg/2017/10/sporean-randomly-leaves-handwritten-calligraphy-notes-all-around-spore-for-strangers/
https://mothership.sg/2017/10/smoking-under-shelter-nea/
https://mothership.sg/2017/10/katongs-name-comes-from-a-sea-mirage-you-see-when-looking-at-the-shoreline/
In [ ]:
Content source: methylDragon/news-anaCrawler
Similar notebooks:
notebook.community | gallery | about