This is a project dealing with the website http://centralkids.org.nz. I just want to play with it and see what I can do.
My first step was downloading the site with wget. Adding source control, a new external folder to hold a new website/data/notebooks (keep everything clean and build it up), and a ipython notebook/py file that is used to generate the new site. This notebook currently sits in the repo main dir.
The script starts by looking at the main dir and filtering the list of folders and files. I do this many times.
Over use of lists. Attempting to use more dict.
Ideas fo
In [456]:
from IPython.display import display, Javascript
def markdown_below():
display(Javascript("""
IPython.notebook.insert_cell_below('markdown')
"""));
markdown_below()
In [457]:
from shutil import make_archive
testing one two three
In [458]:
import os
import re
In [459]:
listkindy = list(os.listdir(os.getcwd()))
In [460]:
for kin in listkindy:
print kin
In [461]:
for kin in listkindy:
print kin
In [462]:
listen = []
In [463]:
savlis = []
In [464]:
for kin in listkindy:
if 'kinder' in kin:
print kin
listen.append(kin)
In [464]:
In [465]:
for kin in listkindy:
if 'early' in kin:
print kin
listen.append(kin)
In [466]:
#listen.sort()
In [467]:
for itz in listen:
if 'html' in itz:
print itz
listen.remove(itz)
In [468]:
import re
In [469]:
for lix in listen:
print lix
savlis.append(geopy.Location(lix))
In [470]:
s = 'abc-d-ef ghijk'
s1 = s.replace('-', ' ')
In [570]:
splist = []
In [571]:
oplist = []
In [572]:
for spz in listen:
savall = open('/home/public/' + spz + '/index.html')
oplist.append(savall.read())
#oplist.append(spz)
In [573]:
for iz in listen:
print iz
splist.append(iz.replace('-', ' '))
In [475]:
caplis = []
In [476]:
for caz in splist:
caplis.append(str.capitalize(caz))
In [477]:
chars = []
In [478]:
for line in caplis:
for c in line:
chars.append(c)
In [479]:
for car in chars:
print car
In [480]:
geolz = []
In [482]:
for iz in caplis:
print iz
geolz.append(geopy.Location(iz))
In [483]:
openhtm = open('index.html', 'r')
In [484]:
mydoc = str(openhtm.read())
In [485]:
mydoc
Out[485]:
In [486]:
from bs4 import BeautifulSoup
In [487]:
cent = BeautifulSoup(mydoc)
In [488]:
ctent = cent.title
In [489]:
pcent = cent.find_all('p')
In [490]:
cenimg = cent.findAll('img')
In [491]:
import dominate
from dominate.tags import *
In [492]:
dominate.tags
Out[492]:
In [493]:
allcent = dominate.document(title='central')
In [494]:
with allcent.head:
link(rel='stylesheet', href='style.css')
script(type='text/javascript', scr='script.js')
In [495]:
(pcent)
Out[495]:
In [496]:
for pcz in pcent:
print pcz
In [497]:
for cen in cenimg:
print cen
In [498]:
with allcent:
with div(id='header').add(ol()):
for i in listen:
li(a(i.title(), href='/%s' % i))
with div():
attr(cls='body')
#p(str(pcent))
for pcz in pcent:
p(pcz)
#p(str(pcent))
In [499]:
print allcent
In [564]:
mkhtm = open('test.html', 'w')
mkhtm.write(str(splist))
mkhtm.close()
In [565]:
cendict = {}
In [566]:
mynum = 0
In [567]:
splist
Out[567]:
In [568]:
for iza in splist:
if 'kindergarten' or 'centre' in iza:
cendict.update({iza: mynum})
mynum = (mynum + 1)
In [569]:
cendict
Out[569]:
In [557]:
import re
In [505]:
a = re.sub('[.!,;]', 'bssdg!aassada.', a)
In [506]:
re.sub()
In [507]:
s = 'A.B-!C?'
print s.translate(None, '" "'.join('-'))
In [508]:
import string
specials = '-"/.' #etc
trans = string.maketrans(specials, ' '*len(specials))
#for line in file
cleanline = 'test'.translate(trans)
In [509]:
s = 'abc-d-ef ghijk'
s1 = s.replace('-', ' ')
dict. kindergartens: list all the kindergarten, centre: list all centre
In [510]:
os.listdir('/home/public/whai/imgs')
Out[510]:
make folder of kindyname-index.html
In [512]:
os.removedirs('/home/public/central-test')
In [513]:
cents = ('/home/public/central-test/centres')
In [514]:
os.mkdir('imgs')
In [515]:
os.listdir('/home/public/whai/imgs/')
Out[515]:
In [516]:
osliz = []
In [517]:
os.chdir(cents)
this makes the centre folders
In [518]:
for oubz in listen:
os.mkdir(oubz)
In [519]:
for oslis in listen:
print oslis
osliz.append(os.listdir('/home/public/whai/' + oslis))
In [520]:
osliz
Out[520]:
In [521]:
finlis = []
In [522]:
for osl in osliz:
finlis.append(osl)
if 'index' in osl:
print 'yes!'
In [523]:
indlis = []
In [524]:
for fin in finlis:
if 'index' in fin:
indlis.append(fin)
In [525]:
telis = []
In [526]:
for f in finlis:
#print f
for te in f:
print te
if 'index.html' or 'wp' in te:
telis.append(te)
In [527]:
filis = []
In [528]:
for tez in telis:
if '.' in tez:
filis.append(tez)
In [529]:
filis
Out[529]:
In [530]:
%%bash
#!/bin/bash
# Add a user to the system, if they don't already exist. This script must be run as root.
# If the user exists in Stanford LDAP, we'll user their Stanford-wide UID. Otherwise,
# they get assigned a local UID that's outside the Stanford UID range.
# Copyright 2012 by Gunnaer Schaefer (gsfr@stanford.edu) and Bob Dougherty (bobd@stanford.edu)
if [ ! $1 ]; then
echo "Usage: $0 SUNetID"
exit 999
fi
if id $1 &> /dev/null ; then
if [ -z "`groups $1 | grep ipython`" ]; then
/usr/sbin/adduser $1 ipython
else
echo "User $1 already exists and is a member of the ipython group."
fi
exit 1
else
ldapinfo=$(ldapsearch -x -h ldap.stanford.edu uid=$1)
uid_num=$(echo "$ldapinfo" | grep uidNumber); uid_num=${uid_num##*: }
firstname=$(echo "$ldapinfo" | grep suDisplayNameFirst); firstname=${firstname##*: }
lastname=$(echo "$ldapinfo" | grep suDisplayNameLast); lastname=${lastname##*: }
if [ -z $uid_num ]; then
echo "User $1 does not exist in LDAP. Assigning a local UID."
uid_num=$((`(echo 69999; cut -d':' -f3 /etc/passwd) | sort -n | tail -1` + 1))
fi
echo "Creating user $1 ($firstname $lastname, uid = $uid_num)..."
/usr/sbin/adduser --disabled-password --uid $uid_num --gecos "$firstname $lastname" $1
/usr/sbin/adduser $1 ipython
fi
In [530]:
makes folder - centre kindy - had folder of all kindys - inside of this is static blog - linked to main centre kindy blog. admin - roll generates a list of students at centre generates a list of teachers at centre warnings on ratio communication with other centre each centre has static page with info about each centre - open hours, list of teachers, job/opotunity, central kids os: linux debian 7 os locationed inside thieir folder. This is based off Debian7 (root7.tar.gz install7). All Central kids computers have this os. Unable to access certain areas outside of os (rsa ssh from inside central kids os into secure server) Ways to keep privacy safe i have no idea what im doing, but i hope something works!
In [531]:
os.chdir('/home/public/')
In [532]:
os.mkdir('central-test')
In [533]:
os.chdir('central-test')
In [581]:
import delican
In [582]:
import tarfile
In [583]:
mytar = tarfile.TarFile('/home/public/central-test/centralblog.tar.gz')
In [ ]:
import shutil
In [ ]:
In [ ]:
shutil.copytree('/home/public/central-test/central/' '/home/public/central-test/waipahihi-kindergarten/')
In [647]:
for ipz in os.listdir('/home/public/central-test'):
#print ipz
os.chdir('/home/public/central-test/' + ipz )
cfpath, cfname = os.path.split(os.getcwd())
print cfname
os.chdir('/home/public/central-test')
In [585]:
import dominate
In [577]:
dominate.tags(p('test'))
In [578]:
current_folder_path, current_folder_name = os.path.split(os.getcwd())
In [579]:
current_folder_name
Out[579]:
In [580]:
centar = '/home/public/central.tar.gz'
centes = '/home/public/central-test/'
In [545]:
for imz in os.listdir('/home/public/central-test'):
print imz
os.chdir(imz)
shutil.copyfile(centar, '/home/public/central-test/whaihanga-early-learning-centre/test.tar.gz')
In [546]:
logincos =
In [547]:
os.urandom(512)
Out[547]:
In [548]:
import os
archive_name = os.path.expanduser(os.path.join('~', 'central'))
root_dir = os.path.expanduser(os.path.join('~', 'central-test/central'))
make_archive(archive_name, 'gztar', root_dir)
Out[548]:
In [660]:
In [661]:
shutil.copyfile('/home/public/central.tar.gz', '/home/public/central-test/whaihanga-early-learning-centre/test.tar.gz')
In [663]:
import tarfile
# tar file to extract
theTarFile = ('/home/public/central-test/whaihanga-early-learning-centre/test.tar.gz')
# tar file path to extract
extractTarPath = ('/home/public/central-test/whaihanga-early-learning-centre/')
# open the tar file
tfile = tarfile.open(theTarFile)
if tarfile.is_tarfile(theTarFile):
# list all contents
print "tar file contents:"
print tfile.list(verbose=False)
# extract all contents
for imz in os.listdir('/home/public/central-test'):
print imz
#os.chdir(imz)
#shutil.copyfile(tfile, '/home/public/central-test/' + imz + '/test.tar.gz')
tfile.extractall('/home/public/central-test/' + imz)
else:
print theTarFile + " is not a tarfile."
In [555]:
cendict
Out[555]:
Parse the blog, because - why not?
In [588]:
import feedparser
In [589]:
cenblog = feedparser.parse('http://feeds.feedburner.com/CentralKidsKindergartens')
In [ ]:
http://feeds.feedburner.com/CentralKidsKindergartens
In [596]:
lenblog = len(cenblog['entries'])
In [597]:
lenblog
Out[597]:
I want to write the blog posts to html file with dominate. it needs to take the print cen['title', ['updated'], ['summary_details'] etc... and make it be be formated nice in html.
In [637]:
blogplz = dominate.document(title='Centre Kids News')
with doc.head:
link(rel='stylesheet', href='style.css')
script(type='text/javascript', src='script.js')
with blogplz:
with div(id='header').add(ol()):
for cen in cenblog['entries']:
#print cen
print cen['title']
li(a(cen['title'](), href='/%s.html' % cen['title']))
print cen['updated']
li(a(cen['updated'](), href='/%s.html' % cen['updated']))
print cen['summary_detail']['value']
li(a(cen['summary_detail'](), href='/%s.html' % cen['summary_detail']))
print cen['author']
li(a(cen['author'](), href='/%s.html' % cen['author']))
#for i in ['home', 'about', 'contact']:
# li(a(i.title(), href='/%s.html' % i))
with div():
attr(cls='body')
p('Lorem ipsum..')
print blogplz
In [621]:
tilis = []
In [623]:
tilis
Out[623]:
In [622]:
with allcent:
with div(id='header').add(ol()):
for i in listen:
li(a(i.title(), href='/%s' % i))
with div():
attr(cls='body')
#p(str(pcent))
for cen in cenblog['entries']:
#print cen
print cen['title']
tilis.append(cen['title'])
print cen['updated']
tilis.append(cen['updated'])
print cen['summary_detail']['value']
tilis.append(cen['summary_detail']['value'])
print cen['author']
tilis.append(cen['author'])
#print cen['value']
#p(str(pcent))
with div():
fo
In [638]:
def getallpost():
for cen in cenblog['entries']:
#print cen
print cen['title']
print cen['updated']
print cen['summary_detail']['value']
print cen['author']
#print cen['value']
In [648]:
%%javascript
<script type='text/javascript' src='http://centralkids.org.nz/wp-content/plugins/wp-google-maps/js/jquery.dataTables.js?ver=3.9.1'></script>
<script type='text/javascript'>
/* <![CDATA[ */
var wpgmaps_localize = {"1":{"id":"1","map_title":"Central North Island Region Map","map_width":"940","map_height":"1100","map_start_lat":"-37.888943","map_start_lng":"176.264919","map_start_location":"-37.88894301874612,176.26491864062496","map_start_zoom":"9","default_marker":"","type":"1","alignment":"1","directions_enabled":"2","styling_enabled":"0","styling_json":"","active":"0","kml":"","bicycle":"2","traffic":"2","dbox":"1","dbox_width":"","listmarkers":"0","listmarkers_advanced":"0","ugm_enabled":"0","fusion":"","map_width_type":"px","map_height_type":"px","mass_marker_support":"1","filterbycat":"0","ugm_access":"0","order_markers_by":"1","order_markers_choice":"2","show_user_location":"2","ugm_category_enabled":"0","default_to":"","other_settings":""}};
var wpgmaps_localize_mashup_ids = null;
var wpgmaps_localize_cat_ids = {"1":"all"};
var wpgmaps_localize_global_settings = {"wpgmza_settings_image_width":"100","wpgmza_settings_image_height":"100","wpgmza_settings_use_timthumb":"","wpgmza_settings_infowindow_width":"200","wpgmza_settings_infowindow_links":"","wpgmza_settings_infowindow_address":"","wpgmza_settings_infowindow_link_text":"More details","wpgmza_settings_map_streetview":"","wpgmza_settings_map_zoom":"","wpgmza_settings_map_pan":"","wpgmza_settings_map_type":"","wpgmza_settings_map_scroll":"","wpgmza_settings_map_draggable":"","wpgmza_settings_map_clickzoom":"","wpgmza_settings_ugm_striptags":"","wpgmza_settings_force_jquery":"yes","wpgmza_settings_markerlist_category":"","wpgmza_settings_markerlist_icon":"","wpgmza_settings_markerlist_title":"","wpgmza_settings_markerlist_address":"","wpgmza_settings_markerlist_description":""};
var wpgmaps_localize_polygon_settings = [];
var wpgmaps_localize_polyline_settings = [];
/* ]]> */
</script>
<script type='text/javascript' src='http://centralkids.org.nz/wp-content/plugins/wp-google-maps-pro/js/core.js?ver=5.27p'></script>
In [646]:
getallpost()
In [598]:
cenblog['entries']
Out[598]: