SeriesStatus



In [1]:
cd git/SolarDataRESTfulAPI/


/root/git/SolarDataRESTfulAPI

In [2]:
ls


cvs_importer.py                 InfluxDBInterface.py.save.1       SiteIDs.json
ElasticsearchInterface.py       influxInterfaceCredentials2.json  SLB_DataImporter.py*
ElasticsearchInterface.pyc      influxInterfaceCredentials.json   SLB_DataImporter.pyc
ElasticsearchInterface.py.save  influx.json                       SolarDataRESTapi.py
Feeds.json                      IoTtoolkit.py                     SolarProductionProducer.py*
influx2.json                    IoTtoolkit.pyc                    SolarRestAPI.sh*
InfluxDBInterface.py            LICENSE                           testdata/
InfluxDBInterface.pyc           README.md                         testREST.py
InfluxDBInterface.py.save       SeriesStatus.py*                  Universe.json

In [3]:
import json
import pandas as pd
import InfluxDBInterface
import time
reload(InfluxDBInterface)
from ElasticsearchInterface import ESinterface
import IoTtoolkit
#reload(IoTtoolkit)
import sys
import mosquitto
import os
import argparse

DataLink = InfluxDBInterface.InfluxDBInterface("influxInterfaceCredentials2.json")

LogDB = DataLink.databases[u'SolarLogdata']
ProductionDB = DataLink.databases[u'SolarProductionSites']
AreaDB = DataLink.databases[u'SolarProductionAreas']
Test = DataLink.databases[u'test']

es = ESinterface()

def EpocToDate(timestamp):
    try:
        return time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(timestamp))
    except:
        return "-"

def SecToHMS(sec):
    try:
        sec = int(sec)
        hour = sec / 3600
        minutes = (sec - (hour * 3600))/60
        secs = sec % 60
        return "%i h %i min %i s" %(hour,minutes,secs)
    except:
        return "-"

In [4]:
def GetRawStatusFrame():

    #Sites = LogDB.ListSeries()
    res = es.GetHitsAsDataFrame(index="solar-sites-index", doc_type='meta-data')
    
    Sites = res.columns
    
    print "Procssing"
    
    
    
    df = pd.DataFrame()
    df.index.name = "Sites"
    
    now = time.time() *1000
    
    for site in Sites:
        print "*",
        
        df.loc[site,"OpID"] = res[site]["Operator_ID"]
        
        df.loc[site,"LogStart"] = LogDB.GetFirstTimestamp(site)
        df.loc[site,"LogStop"] = LogDB.GetLastTimestamp(site)
        
        df.loc[site,"ProdStart"]  = ProductionDB.GetFirstTimestamp(site) 
        df.loc[site,"ProdStop"] = ProductionDB.GetLastTimestamp(site)
    
    
    df["LogLag"] = now - df["LogStop"]
    df["ProdLag"] = now - df["ProdStop"]
    
    print "\rDone!           "
    
    
    return df

In [5]:
def FormatStatusFrame(df):
    df2 = pd.DataFrame()
    
    df2["OpID"] = df["OpID"]
    
    df2["LogLag"] = (df.LogLag/1000).apply(SecToHMS)
    df2["LogStop"] = (df.LogStop/1000).apply(EpocToDate)
    df2["LogStart"] = (df.LogStart/1000).apply(EpocToDate)
    
    df2["ProdLag"] = (df.ProdLag/1000).apply(SecToHMS)
    df2["ProdStop"] = (df.ProdStop/1000).apply(EpocToDate)
    df2["ProdStart"] = (df.ProdStart/1000).apply(EpocToDate)
    
    df2 = df2.reindex_axis(df.columns, axis=1)
    
    return df2

In [6]:
def GetStatusFrame():
    return FormatStatusFrame(GetRawStatusFrame())

In [7]:
def UpdateStatusWebpage(filename = "/var/www/html/status.html"):
    
    status_raw = GetRawStatusFrame()
    status = FormatStatusFrame(status_raw.sort("ProdLag"))
    
    html = status.to_html(max_rows=50)
    
    file = open(filename,"w")
    
    file.write(html)
    
    file.close()
    
    #print status
    
    print "Status written to %s" % filename

In [8]:
UpdateStatusWebpage()


Got 47 Hits:
Procssing
Done!           
Status written to /var/www/html/status.html

In [22]:
status_raw = GetRawStatusFrame()


Got 46 Hits:
Procssing
Done!           

In [23]:
status = FormatStatusFrame(status_raw.sort("LogLag"))

In [123]:
status


Out[123]:
OpID LogStart LogStop ProdStart ProdStop LogLag ProdLag
Sites
ed214e2a-9679-11e4-b100-123b93f75cba b22 2014-11-25 00:00:00 2015-01-08 08:50:00 2014-11-25 00:00:00 2015-01-07 14:30:00 0 h 8 min 54 s 18 h 28 min 54 s
ed21b360-9679-11e4-b100-123b93f75cba b26 2014-11-25 00:00:00 2015-01-08 08:40:00 2014-11-25 00:00:00 2015-01-07 13:10:00 0 h 18 min 54 s 19 h 48 min 54 s
ed21b4e6-9679-11e4-b100-123b93f75cba b27 2014-11-25 00:00:00 2015-01-08 08:40:00 2014-11-25 00:00:00 2015-01-06 03:00:00 0 h 18 min 54 s 53 h 58 min 54 s
ed214bdc-9679-11e4-b100-123b93f75cba b21 2014-11-25 00:00:00 2015-01-08 08:40:00 2014-11-25 00:00:00 2015-01-07 13:30:00 0 h 18 min 54 s 19 h 28 min 54 s
ed2145ce-9679-11e4-b100-123b93f75cba k01 2014-11-15 01:00:00 2015-01-08 08:40:00 2014-11-15 01:00:00 2015-01-08 08:40:00 0 h 18 min 54 s 0 h 18 min 54 s
ed21b1e4-9679-11e4-b100-123b93f75cba b25 2014-11-25 00:00:00 2015-01-08 08:40:00 2014-11-25 00:00:00 2015-01-07 13:20:00 0 h 18 min 54 s 19 h 38 min 54 s
ed21b7fc-9679-11e4-b100-123b93f75cba b29 2014-11-25 00:00:00 2015-01-08 08:40:00 2014-11-25 00:00:00 2015-01-07 13:20:00 0 h 18 min 54 s 19 h 38 min 54 s
ed2149d4-9679-11e4-b100-123b93f75cba b20 2014-11-25 00:00:00 2015-01-08 08:40:00 2014-11-25 00:00:00 2015-01-07 13:10:00 0 h 18 min 54 s 19 h 48 min 54 s
ed21b040-9679-11e4-b100-123b93f75cba b23 2014-11-28 01:00:00 2015-01-08 08:40:00 2014-11-28 01:00:00 2015-01-07 14:30:00 0 h 18 min 54 s 18 h 28 min 54 s
ed21ae9c-9679-11e4-b100-123b93f75cba b24 2014-11-24 00:00:00 2015-01-08 08:10:00 2014-11-24 00:00:00 2015-01-08 08:00:00 0 h 48 min 54 s 0 h 58 min 54 s
79ea7d0e-1da8-11e4-a510-f23c9173ce4a b09 2014-06-09 00:00:00 2015-01-07 17:20:00 2014-06-09 03:20:00 2015-01-07 14:40:00 15 h 38 min 54 s 18 h 18 min 54 s
916b6e8e-1da8-11e4-a510-f23c9173ce4a b12 2014-06-12 09:00:00 2015-01-07 17:20:00 2014-06-12 09:10:00 2015-01-07 14:50:00 15 h 38 min 54 s 18 h 8 min 54 s
709e47a1-ca88-4c22-890f-2407e9cb131a f06 2014-04-08 06:10:00 2015-01-07 15:50:00 2014-04-08 06:10:00 2015-01-07 13:20:00 17 h 8 min 54 s 19 h 38 min 54 s
7f3c36b0-44a6-11e4-916c-0800200c9a66 b16 2014-07-01 00:00:00 2015-01-07 15:50:00 2014-07-01 03:20:00 2015-01-07 14:10:00 17 h 8 min 54 s 18 h 48 min 54 s
b0dd71a8-efe8-41eb-bab1-633ebaaa778c f07 2014-04-10 10:30:00 2015-01-07 15:50:00 2014-04-10 10:30:00 2015-01-07 13:40:00 17 h 8 min 54 s 19 h 18 min 54 s
e93dc809-3491-11e4-8c21-0800200c9a66 b18 2014-07-06 08:00:00 2015-01-07 15:40:00 2014-07-06 08:10:00 2015-01-07 14:40:00 17 h 18 min 54 s 18 h 18 min 54 s
4cf6c743-8d0b-45f0-aa72-8a0c10315cf5 f09 2014-03-26 05:30:00 2015-01-07 15:40:00 2014-03-26 05:30:00 2015-01-07 13:20:00 17 h 18 min 54 s 19 h 38 min 54 s
845e5e54-1da8-11e4-a510-f23c9173ce4a b10 2014-06-11 17:20:00 2015-01-07 15:40:00 2014-06-11 17:30:00 2015-01-07 14:10:00 17 h 18 min 54 s 18 h 48 min 54 s
8b28b202-1da8-11e4-a510-f23c9173ce4a b11 2014-06-11 19:10:00 2015-01-07 15:40:00 2014-06-11 19:20:00 2015-01-07 13:50:00 17 h 18 min 54 s 19 h 8 min 54 s
71b0e5ec-1da8-11e4-a510-f23c9173ce4a b08 2014-06-09 17:00:00 2015-01-07 15:40:00 2014-06-09 17:20:00 2015-01-07 14:40:00 17 h 18 min 54 s 18 h 18 min 54 s
7445f840-44a6-11e4-916c-0800200c9a66 b15 2014-07-02 14:10:00 2015-01-07 15:40:00 2014-07-02 14:50:00 2015-01-07 13:20:00 17 h 18 min 54 s 19 h 38 min 54 s
6f075b69-b823-4e6b-8021-dd751cc79eea b03 2014-03-05 12:30:00 2015-01-07 15:20:00 2014-03-05 12:30:00 2015-01-07 13:20:00 17 h 38 min 54 s 19 h 38 min 54 s
32383bde-7648-4abe-9dac-44701eabd72f f04 2014-04-09 06:10:00 2015-01-07 15:20:00 2014-04-09 06:10:00 2015-01-07 13:20:00 17 h 38 min 54 s 19 h 38 min 54 s
e8a13982-f651-11e3-a510-f23c9173ce4a f11 2014-04-07 06:00:00 2015-01-07 15:20:00 2014-04-07 06:00:00 2015-01-07 13:50:00 17 h 38 min 54 s 19 h 8 min 54 s
4a39b124-f594-11e3-a510-f23c9173ce4a b02 2014-06-06 03:30:00 2015-01-07 15:20:00 2014-06-06 03:30:00 2015-01-07 13:20:00 17 h 38 min 54 s 19 h 38 min 54 s
2a31fb24-347b-4924-ab89-5c434771a2ae b06 2014-03-05 17:00:00 2015-01-07 15:00:00 2014-03-05 17:00:00 2015-01-07 13:20:00 17 h 58 min 54 s 19 h 38 min 54 s
f1136e33-d227-4fbd-ae72-d3488d70ba89 b01 2014-03-06 08:10:00 2015-01-07 15:00:00 2014-03-06 08:10:00 2015-01-07 12:50:00 17 h 58 min 54 s 20 h 8 min 54 s
802afb51-c5eb-4230-9b22-9a77ef7260b3 b05 2014-03-05 16:30:00 2015-01-07 15:00:00 2014-03-05 16:30:00 2015-01-07 13:20:00 17 h 58 min 54 s 19 h 38 min 54 s
46d55815-f927-459f-a8e2-8bbcd88008ee h00 2013-10-16 07:20:00 2015-01-07 14:40:00 2013-10-16 07:20:00 2015-01-07 14:10:00 18 h 18 min 54 s 18 h 48 min 54 s
c6261166-f651-11e3-a510-f23c9173ce4a f10 2014-04-07 06:00:00 2015-01-07 11:40:00 2014-04-07 06:00:00 2015-01-07 11:40:00 21 h 18 min 54 s 21 h 18 min 54 s
aa3695b0-967a-11e4-b100-123b93f75cba f14 2014-11-25 13:30:00 2014-12-29 10:00:00 2014-11-25 13:30:00 2014-12-23 07:00:00 238 h 58 min 54 s 385 h 58 min 54 s
aa36a136-967a-11e4-b100-123b93f75cba f21 2014-11-25 10:00:00 2014-12-29 10:00:00 2014-11-25 10:00:00 2014-12-28 14:30:00 238 h 58 min 54 s 258 h 28 min 54 s
aa369b78-967a-11e4-b100-123b93f75cba f18 2014-11-25 10:00:00 2014-12-29 10:00:00 2014-11-25 10:00:00 2014-12-28 14:50:00 238 h 58 min 54 s 258 h 8 min 54 s
aa369330-967a-11e4-b100-123b93f75cba f13 2014-11-25 13:30:00 2014-12-29 10:00:00 2014-11-25 13:30:00 2014-12-28 14:10:00 238 h 58 min 54 s 258 h 48 min 54 s
aa369fd8-967a-11e4-b100-123b93f75cba f19 2014-11-25 10:00:00 2014-12-29 10:00:00 2014-11-25 10:00:00 2014-12-28 14:40:00 238 h 58 min 54 s 258 h 18 min 54 s
aa369a4c-967a-11e4-b100-123b93f75cba f17 2014-11-25 10:00:00 2014-12-29 10:00:00 2014-11-25 10:00:00 2014-12-27 10:50:00 238 h 58 min 54 s 286 h 8 min 54 s
aa369920-967a-11e4-b100-123b93f75cba f16 2014-11-25 10:00:00 2014-12-29 10:00:00 2014-11-25 10:00:00 2014-12-28 14:20:00 238 h 58 min 54 s 258 h 38 min 54 s
aa3697b8-967a-11e4-b100-123b93f75cba f15 2014-11-25 13:30:00 2014-12-29 10:00:00 2014-11-25 13:30:00 2014-12-28 14:50:00 238 h 58 min 54 s 258 h 8 min 54 s
aa36a5a0-967a-11e4-b100-123b93f75cba f20 2014-11-25 10:00:00 2014-12-29 10:00:00 2014-11-25 10:00:00 2014-12-28 14:40:00 238 h 58 min 54 s 258 h 18 min 54 s
aa36a348-967a-11e4-b100-123b93f75cba f22 2014-11-25 01:00:00 2014-12-29 10:00:00 2014-11-25 01:00:00 2014-12-28 14:40:00 238 h 58 min 54 s 258 h 18 min 54 s
aa368f3e-967a-11e4-b100-123b93f75cba f12 2014-11-25 13:30:00 2014-12-29 10:00:00 2014-11-25 13:30:00 2014-12-23 07:00:00 238 h 58 min 54 s 385 h 58 min 54 s
ed21b6d0-9679-11e4-b100-123b93f75cba b28 2014-11-24 14:00:00 2014-12-12 00:00:00 2014-11-24 14:00:00 2014-12-11 23:00:00 656 h 58 min 54 s 657 h 58 min 54 s
d9c86a10-3491-11e4-8c21-0800200c9a66 b17 2014-07-02 10:00:00 2014-12-09 12:30:00 2014-07-02 10:10:00 2014-12-09 12:30:00 716 h 28 min 54 s 716 h 28 min 54 s
beeb4730-3491-11e4-8c21-0800200c9a66 b14 2014-07-02 11:00:00 2014-12-09 12:30:00 2014-07-02 11:10:00 2014-12-09 12:30:00 716 h 28 min 54 s 716 h 28 min 54 s
f22857f0-3491-11e4-8c21-0800200c9a66 b19 2014-07-01 19:00:00 2014-12-08 14:20:00 2014-07-01 19:10:00 2014-12-08 14:20:00 738 h 38 min 54 s 738 h 38 min 54 s
face0d13-b22d-48fe-bec1-d29732604133 f05 2014-04-08 06:10:00 2014-09-30 09:20:00 2014-04-08 06:10:00 2014-09-30 01:30:00 2400 h 38 min 54 s 2408 h 28 min 54 s

In [10]:
slb = pd.io.html.read_html("http://slb.nu/soldata/")

In [12]:
type(slb)


Out[12]:
list

In [36]:
ids = slb[0][8].dropna()

In [37]:
ids


Out[37]:
3     stnKEY
4        b01
5        b02
6        b03
7        b04
8        b05
10       b06
11       b08
12       b09
13       b10
14       b11
16       b12
17       b14
18       b15
19       b16
20       b17
22       b18
23       b19
24       b20
25       b21
26       b22
28       b23
29       b24
30       b25
31       b26
32       b27
34       b28
35       b29
36       f04
37       f05
38       f06
40       f07
41       f09
42       f10
43       f11
44       f12
46       f13
47       f14
48       f15
49       f16
50       f17
52       f18
53       f19
54       f20
55       f21
56       f22
58       h00
59       k01
Name: 8, dtype: object

In [24]:
id2 = status["OpID"]

In [26]:
ids2 = id2.values

In [28]:
ids2


Out[28]:
array([u'b10', u'b29', u'b23', u'b21', u'b25', u'b27', u'b02', u'b01',
       u'b09', u'b12', u'b06', u'b05', u'b22', u'b03', u'k01', u'b24',
       u'f09', u'f04', u'b26', u'f11', u'b20', u'h00', u'f06', u'f07',
       u'f10', u'b11', u'b15', u'b18', u'b16', u'b08', u'f19', u'f18',
       u'f13', u'f14', u'f12', u'f16', u'f15', u'f20', u'f17', u'f22',
       u'f21', u'b28', u'b14', u'b17', u'b19', u'f05'], dtype=object)

In [38]:
ids = ids.values

In [35]:
ids

In [32]:
ids = ids.sort()

In [33]:
dis


---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-33-48f87b3eeed4> in <module>()
----> 1 dis

NameError: name 'dis' is not defined

In [39]:
ids.sort()

In [40]:
ids


Out[40]:
array([u'b01', u'b02', u'b03', u'b04', u'b05', u'b06', u'b08', u'b09',
       u'b10', u'b11', u'b12', u'b14', u'b15', u'b16', u'b17', u'b18',
       u'b19', u'b20', u'b21', u'b22', u'b23', u'b24', u'b25', u'b26',
       u'b27', u'b28', u'b29', u'f04', u'f05', u'f06', u'f07', u'f09',
       u'f10', u'f11', u'f12', u'f13', u'f14', u'f15', u'f16', u'f17',
       u'f18', u'f19', u'f20', u'f21', u'f22', u'h00', u'k01', u'stnKEY'], dtype=object)

In [41]:
ids2.sort()

In [42]:
ids2


Out[42]:
array([u'b01', u'b02', u'b03', u'b05', u'b06', u'b08', u'b09', u'b10',
       u'b11', u'b12', u'b14', u'b15', u'b16', u'b17', u'b18', u'b19',
       u'b20', u'b21', u'b22', u'b23', u'b24', u'b25', u'b26', u'b27',
       u'b28', u'b29', u'f04', u'f05', u'f06', u'f07', u'f09', u'f10',
       u'f11', u'f12', u'f13', u'f14', u'f15', u'f16', u'f17', u'f18',
       u'f19', u'f20', u'f21', u'f22', u'h00', u'k01'], dtype=object)

In [ ]: