notebook.community

Edit and run 





In [2]:



    


'''
This code updates the smaller.db database.
To run this file, open a command prompt in this folder and run the command
python update_database.py
If you have any difficulties or questions, contact Max Vilgalys (vilgalys@mit.edu)
'''

print("Determining maximum date in database...")
# First find how current the database is - this finds the most recent month of data in 
# smaller.db, so it won't pull any more recent data.
import sqlite3
import pandas
con = sqlite3.connect('smaller.db')
max_date = pandas.read_sql('select max(op_date) from data', con)
for i, row in max_date.iterrows():
    date = str(row[0])
max_month = date[0:2]
max_year = date[-4:]
print("Max date found: " + max_month + " " + max_year)

print()
print("Downloading additional zip files...")

from ftplib import FTP
import os.path

import zipfile
from pathlib2 import Path

ftp = FTP('ftp.epa.gov', timeout=60)
ftp.login()
ftp.cwd('dmdnload')
ftp.cwd('emissions')
ftp.cwd('hourly')
ftp.cwd('monthly')
years = []
ftp.retrlines('NLST', years.append)
parent_directory = ftp.pwd()

def zip_fetch(ftp, entry):
    print("Now fetching: " + entry)
    outfile = open('data/' + entry, 'wb')
    ftp.retrbinary('RETR ' + entry, outfile.write)
    outfile.close()
    return
states = ['wa', 'or', 'ca', 'id', 'nv', 'ut', 'az', 'nm', 'co', 'wy', 'mt', 'tx']
def inWest(string):
    return string[4:6] in states

if not os.path.exists('data'):
    os.makedirs('data')


def unzip(name):
    with zipfile.ZipFile(name, "r") as z:
        z.extractall('unzipped')

if not os.path.exists('unzipped'):
    os.makedirs('unzipped')

for year in years:
    if (int(year) < int(max_year)): continue
    ftp.cwd(str(year))
    files = []
    ftp.retrlines('NLST', files.append)
    for entry in files:
        if (int(year) == int(max_year) and int(entry[6:8]) <= int(max_month)) or not inWest(entry):
            continue
        try:
            zip_fetch(ftp, entry)

        except:
            pass
    for n_month in range(1, 13):		
        for state in ['wa', 'or', 'ca', 'id', 'nv', 'ut', 'az', 'nm', 'co', 'wy', 'mt', 'tx']:
            month = str(n_month) if n_month > 9 else "0" + str(n_month)
            name = "data/" + str(year) + state + month + ".zip"
            if os.path.isfile(name):
            	print("Now unzipping: " + name )
                unzip(name)           
    ftp.cwd('..')



    


















    






Determining maximum date in database...
Max date found: 12 2016
()
Downloading additional zip files...
Now unzipping: data/2016wa01.zip
Now unzipping: data/2016or01.zip
Now unzipping: data/2016ca01.zip
Now unzipping: data/2016id01.zip
Now unzipping: data/2016nv01.zip
Now unzipping: data/2016ut01.zip
Now unzipping: data/2016az01.zip
Now unzipping: data/2016nm01.zip
Now unzipping: data/2016co01.zip
Now unzipping: data/2016wy01.zip
Now unzipping: data/2016mt01.zip
Now unzipping: data/2016tx01.zip
Now unzipping: data/2016wa02.zip
Now unzipping: data/2016or02.zip
Now unzipping: data/2016ca02.zip
Now unzipping: data/2016id02.zip
Now unzipping: data/2016nv02.zip
Now unzipping: data/2016ut02.zip
Now unzipping: data/2016az02.zip
Now unzipping: data/2016nm02.zip
Now unzipping: data/2016co02.zip
Now unzipping: data/2016wy02.zip
Now unzipping: data/2016mt02.zip
Now unzipping: data/2016tx02.zip
Now unzipping: data/2016wa03.zip
Now unzipping: data/2016or03.zip
Now unzipping: data/2016ca03.zip
Now unzipping: data/2016id03.zip
Now unzipping: data/2016nv03.zip
Now unzipping: data/2016ut03.zip
Now unzipping: data/2016az03.zip
Now unzipping: data/2016nm03.zip
Now unzipping: data/2016co03.zip
Now unzipping: data/2016wy03.zip
Now unzipping: data/2016mt03.zip
Now unzipping: data/2016tx03.zip
Now unzipping: data/2016wa04.zip
Now unzipping: data/2016or04.zip
Now unzipping: data/2016ca04.zip
Now unzipping: data/2016id04.zip
Now unzipping: data/2016nv04.zip
Now unzipping: data/2016ut04.zip
Now unzipping: data/2016az04.zip
Now unzipping: data/2016nm04.zip
Now unzipping: data/2016co04.zip
Now unzipping: data/2016wy04.zip
Now unzipping: data/2016mt04.zip
Now unzipping: data/2016tx04.zip
Now unzipping: data/2016wa05.zip
Now unzipping: data/2016or05.zip
Now unzipping: data/2016ca05.zip
Now unzipping: data/2016id05.zip
Now unzipping: data/2016nv05.zip
Now unzipping: data/2016ut05.zip
Now unzipping: data/2016az05.zip
Now unzipping: data/2016nm05.zip
Now unzipping: data/2016co05.zip
Now unzipping: data/2016wy05.zip
Now unzipping: data/2016mt05.zip
Now unzipping: data/2016tx05.zip
Now unzipping: data/2016wa06.zip
Now unzipping: data/2016or06.zip
Now unzipping: data/2016ca06.zip
Now unzipping: data/2016id06.zip
Now unzipping: data/2016nv06.zip
Now unzipping: data/2016ut06.zip
Now unzipping: data/2016az06.zip
Now unzipping: data/2016nm06.zip
Now unzipping: data/2016co06.zip
Now unzipping: data/2016wy06.zip
Now unzipping: data/2016mt06.zip
Now unzipping: data/2016tx06.zip
Now unzipping: data/2016wa07.zip
Now unzipping: data/2016or07.zip
Now unzipping: data/2016ca07.zip
Now unzipping: data/2016id07.zip
Now unzipping: data/2016nv07.zip
Now unzipping: data/2016ut07.zip
Now unzipping: data/2016az07.zip
Now unzipping: data/2016nm07.zip
Now unzipping: data/2016co07.zip
Now unzipping: data/2016wy07.zip
Now unzipping: data/2016mt07.zip
Now unzipping: data/2016tx07.zip
Now unzipping: data/2016wa08.zip
Now unzipping: data/2016or08.zip
Now unzipping: data/2016ca08.zip
Now unzipping: data/2016id08.zip
Now unzipping: data/2016nv08.zip
Now unzipping: data/2016ut08.zip
Now unzipping: data/2016az08.zip
Now unzipping: data/2016nm08.zip
Now unzipping: data/2016co08.zip
Now unzipping: data/2016wy08.zip
Now unzipping: data/2016mt08.zip
Now unzipping: data/2016tx08.zip
Now unzipping: data/2016wa09.zip
Now unzipping: data/2016or09.zip
Now unzipping: data/2016ca09.zip
Now unzipping: data/2016id09.zip
Now unzipping: data/2016nv09.zip
Now unzipping: data/2016ut09.zip
Now unzipping: data/2016az09.zip
Now unzipping: data/2016nm09.zip
Now unzipping: data/2016co09.zip
Now unzipping: data/2016wy09.zip
Now unzipping: data/2016mt09.zip
Now unzipping: data/2016tx09.zip
Now unzipping: data/2016wa10.zip
Now unzipping: data/2016or10.zip
Now unzipping: data/2016ca10.zip
Now unzipping: data/2016id10.zip
Now unzipping: data/2016nv10.zip
Now unzipping: data/2016ut10.zip
Now unzipping: data/2016az10.zip
Now unzipping: data/2016nm10.zip
Now unzipping: data/2016co10.zip
Now unzipping: data/2016wy10.zip
Now unzipping: data/2016mt10.zip
Now unzipping: data/2016tx10.zip
Now unzipping: data/2016wa11.zip
Now unzipping: data/2016or11.zip
Now unzipping: data/2016ca11.zip
Now unzipping: data/2016id11.zip
Now unzipping: data/2016nv11.zip
Now unzipping: data/2016ut11.zip
Now unzipping: data/2016az11.zip
Now unzipping: data/2016nm11.zip
Now unzipping: data/2016co11.zip
Now unzipping: data/2016wy11.zip
Now unzipping: data/2016mt11.zip
Now unzipping: data/2016tx11.zip
Now unzipping: data/2016wa12.zip
Now unzipping: data/2016or12.zip
Now unzipping: data/2016ca12.zip
Now unzipping: data/2016id12.zip
Now unzipping: data/2016nv12.zip
Now unzipping: data/2016ut12.zip
Now unzipping: data/2016az12.zip
Now unzipping: data/2016nm12.zip
Now unzipping: data/2016co12.zip
Now unzipping: data/2016wy12.zip
Now unzipping: data/2016mt12.zip
Now unzipping: data/2016tx12.zip
Now fetching: 2017az01.zip
Now fetching: 2017az02.zip
Now fetching: 2017az03.zip
Now fetching: 2017ca01.zip
Now fetching: 2017ca02.zip
Now fetching: 2017ca03.zip
Now fetching: 2017co01.zip
Now fetching: 2017co02.zip
Now fetching: 2017co03.zip
Now fetching: 2017id01.zip
Now fetching: 2017id02.zip
Now fetching: 2017id03.zip
Now fetching: 2017mt01.zip
Now fetching: 2017mt02.zip
Now fetching: 2017mt03.zip
Now fetching: 2017nm01.zip
Now fetching: 2017nm02.zip
Now fetching: 2017nm03.zip
Now fetching: 2017nv01.zip
Now fetching: 2017nv02.zip
Now fetching: 2017nv03.zip
Now fetching: 2017or01.zip
Now fetching: 2017or02.zip
Now fetching: 2017or03.zip
Now fetching: 2017tx01.zip
Now fetching: 2017tx02.zip
Now fetching: 2017tx03.zip
Now fetching: 2017ut01.zip
Now fetching: 2017ut02.zip
Now fetching: 2017ut03.zip
Now fetching: 2017wa01.zip
Now fetching: 2017wa02.zip
Now fetching: 2017wa03.zip
Now fetching: 2017wy01.zip
Now fetching: 2017wy02.zip
Now fetching: 2017wy03.zip
Now unzipping: data/2017wa01.zip
Now unzipping: data/2017or01.zip
Now unzipping: data/2017ca01.zip
Now unzipping: data/2017id01.zip
Now unzipping: data/2017nv01.zip
Now unzipping: data/2017ut01.zip
Now unzipping: data/2017az01.zip
Now unzipping: data/2017nm01.zip
Now unzipping: data/2017co01.zip
Now unzipping: data/2017wy01.zip
Now unzipping: data/2017mt01.zip
Now unzipping: data/2017tx01.zip
Now unzipping: data/2017wa02.zip
Now unzipping: data/2017or02.zip
Now unzipping: data/2017ca02.zip
Now unzipping: data/2017id02.zip
Now unzipping: data/2017nv02.zip
Now unzipping: data/2017ut02.zip
Now unzipping: data/2017az02.zip
Now unzipping: data/2017nm02.zip
Now unzipping: data/2017co02.zip
Now unzipping: data/2017wy02.zip
Now unzipping: data/2017mt02.zip
Now unzipping: data/2017tx02.zip
Now unzipping: data/2017wa03.zip
Now unzipping: data/2017or03.zip
Now unzipping: data/2017ca03.zip
Now unzipping: data/2017id03.zip
Now unzipping: data/2017nv03.zip
Now unzipping: data/2017ut03.zip
Now unzipping: data/2017az03.zip
Now unzipping: data/2017nm03.zip
Now unzipping: data/2017co03.zip
Now unzipping: data/2017wy03.zip
Now unzipping: data/2017mt03.zip
Now unzipping: data/2017tx03.zip

















In [ ]:

Content source: maxoboe/CEMS_Project

Similar notebooks:

notebook.community | gallery | about