In [1]:

    

import os
import xml.etree.ElementTree as ET
import datetime
import random
import pymongo as mongo
from collections import OrderedDict
import pandas as pd
import OakParkDatasetAdapter as opda


    

    




/usr/local/lib/python2.7/dist-packages/pandas/io/excel.py:626: UserWarning: Installed openpyxl is not supported at this time. Use >=1.6.1 and <2.0.0.
  .format(openpyxl_compat.start_ver, openpyxl_compat.stop_ver))






    




---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-1-66341debb7c9> in <module>()
      6 from collections import OrderedDict
      7 import pandas as pd
----> 8 import OakParkDatasetAdapter as opda

ImportError: No module named OakParkDatasetAdapter

In [26]:

    

reload(opda)


    

    
Out[26]:





<module 'OakParkDatasetAdapter' from 'OakParkDatasetAdapter.py'>

In [8]:

    

##change this part to where your files are
#path = os.path.join(os.getcwd(), 'xml_files')
#xml_files = [f for f in os.listdir(path)]


    

In [23]:

    

def get_tree(xml_file):
    tree = ET.parse('{}'.format(xml_file))
    root = tree.getroot()
    return tree,root


    

In [10]:

    

def get_interval_data(tree_root):
    base_interval_string = './/{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}content/{http://naesb.org/espi}IntervalBlock/'
    S = '{http://naesb.org/espi}'
    interval  = tree_root.findall('{0}/{1}{2}'.format(base_interval_string,S,'interval/'))
    interval_dict = {interval[i].tag.split('{http://naesb.org/espi}')[1]:interval[i].text for i in range(len(interval))} 
    interval_dict['start']=datetime.datetime.fromtimestamp(float(interval_dict['start']))
    return interval_dict


    

In [11]:

    

def get_interval_block(tree_root):
    base_interval_string = './/{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}content/{http://naesb.org/espi}IntervalBlock/'
    S = '{http://naesb.org/espi}'
    reading  = tree_root.findall('{0}/{1}{2}'.format(base_interval_string,S,'IntervalReading/'))
    values = []
    index_date = [] 
    bigger = []
    dict_test = {}
    for r in range(0,len(reading)-1,2):
        index_date.append(datetime.datetime.fromtimestamp(float(reading[r][1].text)))
        values.append(reading[r+1].text)
        bigger.append([datetime.datetime.fromtimestamp(float(reading[r][1].text)),{'value':float(reading[r+1].text)/1e3}])
        #dict_test[str(datetime.datetime.utcfromtimestamp(float(reading[r][1].text)))] =reading[r+1].text     
    return bigger


    

In [12]:

    

def get_currency(tree_root):
    base_interval_string = './/{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}content/{http://naesb.org/espi}ReadingType/'
    S = '{http://naesb.org/espi}'
    currency  = tree_root.findall('{0}/{1}{2}'.format(base_interval_string,S,'currency'))
    return currency[0].text


    

In [13]:

    

def get_address(tree_root):
    return tree_root.findall('{0}'.format('.//{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}title'))[0].text


    

In [19]:

    

def get_meta(tree_root):
    meta_data = {}
    meta_data['updated'] = tree_root.findall('{0}'.format('.//{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}updated'))[0].text
    meta_data['published'] = tree_root.findall('{0}'.format('.//{http://www.w3.org/2005/Atom}entry/{http://www.w3.org/2005/Atom}published'))[0].text
    meta_data['interval'] = get_interval_data(tree_root)
    meta_data['currency'] = get_currency(tree_root)
    meta_data['account_number']=None
    return meta_data


    

In [21]:

    

def get_xml_house(xml_file):
    tree,root = get_tree(xml_file)
    address = get_address(root)
    house ={}
    house['meta'] = get_meta(root)
    house['meta']['dataid']=random.randint(100,9999)
    house['interval_readings'] = (get_interval_block(root))
    return house


    

In [24]:

    

t = 'cec_electric_interval_data_Service 1_2012-07-25_to_2014-07-16.xml'
xml_house = get_xml_house(t)


    

In [27]:

    

trace = opda.get_trace_from_intermediate_xml(xml_house)


    

In [29]:

    

trace.series


    

    
Out[29]:





2012-07-26 00:00:00    0.036
2012-07-26 00:30:00    0.029
2012-07-26 01:00:00    0.034
2012-07-26 01:30:00    0.047
2012-07-26 02:00:00    0.043
2012-07-26 02:30:00    0.027
2012-07-26 03:00:00    0.028
2012-07-26 03:30:00    0.047
2012-07-26 04:00:00    0.045
2012-07-26 04:30:00    0.032
2012-07-26 05:00:00    0.032
2012-07-26 05:30:00    0.030
2012-07-26 06:00:00    0.040
2012-07-26 06:30:00    0.043
2012-07-26 07:00:00    0.043
...
2014-07-15 16:30:00    0.954
2014-07-15 17:00:00    0.395
2014-07-15 17:30:00    0.313
2014-07-15 18:00:00    0.348
2014-07-15 18:30:00    0.322
2014-07-15 19:00:00    0.321
2014-07-15 19:30:00    0.338
2014-07-15 20:00:00    0.261
2014-07-15 20:30:00    0.258
2014-07-15 21:00:00    0.230
2014-07-15 21:30:00    0.351
2014-07-15 22:00:00    0.299
2014-07-15 22:30:00    0.201
2014-07-15 23:00:00    0.201
2014-07-15 23:30:00    0.190
Length: 33678

In [ ]: