In [1]:
%matplotlib inline
%config InlineBackend.figure_format='retina'
from datacharm import *
import matplotlib
import matplotlib.dates as mpd
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
import matplotlib.patches as mp
import datetime as dt
from io import BytesIO
import locale
from enerpi.database import extract_log_file
from enerpi.base import timeit
from enerpi.database import init_catalog
# Pandas to html!
INIT_LOG_MARK = "Init ENERPI logging & broadcasting..."
def _extract_log_file(log_file, extract_temps=True, verbose=True):
rg_log_msg = re.compile('(?P<tipo>INFO|WARNING|DEBUG|ERROR) \[(?P<func>.+?)\] '
'- (?P<ts>\d{1,2}/\d\d/\d\d\d\d \d\d:\d\d:\d\d): (?P<msg>.*?)\n', re.DOTALL)
with open(log_file, 'r') as log_f:
df_log = pd.DataFrame(rg_log_msg.findall(log_f.read()),
columns=['tipo', 'func', 'ts', 'msg'])
df_log.drop('func', axis=1, inplace=True)
df_log['tipo'] = df_log['tipo'].astype('category')
df_log['ts'] = df_log['ts'].apply(lambda x: dt.datetime.strptime(x, '%d/%m/%Y %H:%M:%S'))
df_log.loc[df_log.msg.str.startswith('Tªs --> '), 'temp'] = True
df_log.loc[df_log.msg.str.startswith('SENDED: '), 'debug_send'] = True
b_warn = df_log.tipo == 'WARNING'
df_log.loc[b_warn, 'no_red'] = df_log[b_warn].msg.str.startswith('OSError: [Errno 101] La red es inaccesible')
df_log['exec'] = df_log['msg'].str.contains(INIT_LOG_MARK).cumsum().astype(int)
df_log = df_log.set_index('ts')
if extract_temps:
rg_temps = 'Tªs --> (?P<CPU>\d{1,2}\.\d) / (?P<GPU>\d{1,2}\.\d) ºC'
df_log = df_log.join(df_log[df_log['temp'].notnull()].msg.str.extract(rg_temps, expand=True).astype(float))
if verbose:
clasific = df_log.groupby(['exec', 'tipo']).count().dropna(how='all').astype(int)
print_ok(clasific)
conteo_tipos = df_log.groupby('tipo').count()
if 'ERROR' in conteo_tipos.index:
print_err(df_log[df_log.tipo == 'ERROR'].dropna(how='all', axis=1))
if 'INFO' in conteo_tipos.index:
print_info(df_log[df_log.tipo == 'INFO'].dropna(how='all', axis=1))
return df_log
#log = extract_log_file('/Users/uge/Dropbox/PYTHON/PYPROJECTS/enerpi/enerpi/DATA/enerpi.log', verbose=False)
#ejecuciones = list(sorted(set(log['exec'])))
#print_red(ejecuciones)
#print_info(log.count())
#last = log[log['exec'] == ejecuciones[-1]]
#penult = log[log['exec'] == ejecuciones[-2]]
#last
In [3]:
catalog = init_catalog(base_path='/Users/uge/ENERPIDATA/')
catalog
Out[3]:
In [28]:
# TILES optimization
IMG_BASEPATH = '/Users/uge/ENERPIDATA/PLOTS'
DEFAULT_IMG_MASK = 'enerpi_power_consumption_ldr_{:%Y%m%d_%H%M}_{:%Y%m%d_%H%M}.png'
def _gen_tableau20():
# # These are the "Tableau 20" colors as RGB.
tableau = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),
(44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),
(148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),
(227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),
(188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]
# Scale the RGB values to the [0, 1] range, which is the format matplotlib accepts.
for i in range(len(tableau)):
r, g, b = tableau[i]
tableau[i] = (r / 255., g / 255., b / 255.)
return tableau
# semaforo_4 = [sns.palettes.crayons[k] for k in ['Green', 'Sea Green', 'Mango Tango', 'Razzmatazz']]
# These are the "Tableau 20" colors as RGB.
tableau20 = _gen_tableau20()
lang, codec = locale.getlocale()
use_locale = '{}.{}'.format(lang, codec)
locale.setlocale(locale.LC_ALL, use_locale)
# sns.set_style('whitegrid')
REGEXPR_SVG_HEIGHT = re.compile(r'<svg height="\d{1,4}pt"')
REGEXPR_SVG_WIDTH = re.compile(r' width="(\d{1,4}pt")')
GRIDSPEC_FULL = {'left': 0, 'right': 1, 'bottom': 0, 'top': 1, 'hspace': 0}
# GRIDSPEC_NORMAL = {'left': 0.075, 'right': .925, 'bottom': 0.11, 'top': 0.91, 'hspace': 0}
FONTSIZE = 10
FONTSIZE_TILE = 12
TICK_PARAMS_TILE = dict(direction='in', pad=-15, length=3, width=.5)
font = {'family': 'sans-serif',
'size': FONTSIZE} # 'weight' : 'light',
matplotlib.rc('font', **font)
@timeit('_write_fig_to_svg')
def _write_fig_to_svg(fig, name_img):
#plt.close(fig)
canvas = FigureCanvas(fig)
output = BytesIO()
imgformat = 'svg'
canvas.print_figure(output, format=imgformat, transparent=True)
svg_out = output.getvalue()
# preserve_ratio=True
svg_out = REGEXPR_SVG_WIDTH.sub(' width="100%" preserveAspectRatio="none"',
REGEXPR_SVG_HEIGHT.sub('<svg height="100%"', svg_out.decode(), count=0),
count=0).encode()
try:
with open(name_img, 'wb') as f:
f.write(svg_out)
except Exception as e:
print('HA OCURRIDO UN ERROR GRABANDO SVG A DISCO: {}'.format(e))
return False
return True
def _tile_figsize(fraction=1.):
dpi = 72
# height = 200
# width = 1.875 * height
height = 200
width = 4.5 * height * fraction
return (round(width / dpi, 2), round(height / dpi, 2))
@timeit('_prep_axis_tile')
def _prep_axis_tile(color):
matplotlib.rcParams['axes.linewidth'] = 0
fig, ax = plt.subplots(figsize=_tile_figsize(), dpi=72, gridspec_kw=GRIDSPEC_FULL, facecolor='none')
fig.patch.set_alpha(0)
ax.patch.set_alpha(0)
ax.tick_params(direction='in', pad=-15, length=3, width=.5)
ax.tick_params(axis='y', length=0, width=0, labelsize=FONTSIZE_TILE)
ax.tick_params(axis='x', which='both', top='off', labelbottom='off')
ax.xaxis.grid(True, color=color, linestyle=':', linewidth=1.5, alpha=.6)
ax.yaxis.grid(True, color=color, linestyle=':', linewidth=1, alpha=.5)
return fig, ax
@timeit('_adjust_tile_limits')
def _adjust_tile_limits(name, ylim, date_ini, date_fin, ax):
ax.set_ylim(ylim)
ax.set_xlim(left=date_ini, right=date_fin)
yticks = list(ax.get_yticks())[1:-1]
yticks_l = [v for v in yticks if (v - ylim[0] < (2 * (ylim[1] - ylim[0]) / 3)) and (v > ylim[0])]
ax.set_yticks(yticks)
if name == 'power':
ax.set_yticklabels([str(round(float(y / 1000.), 1)) + 'kW' for y in yticks_l])
ax.tick_params(pad=-45)
elif name == 'ldr':
ax.set_yticklabels([str(round(float(y / 10.))) + '%' for y in yticks_l])
ax.tick_params(pad=-40)
else:
ax.tick_params(pad=-30)
ax.set_yticklabels([str(round(y, 4)) for y in yticks_l])
return ax
@timeit('plot_tile_last_24h')
def plot_tile_last_24h(data_s, rs_data_s=None, rm_data_s=None, barplot=False, ax=None, fig=None):
color = [1, 0, 1]
if ax is None:
fig, ax = _prep_axis_tile(color)
if not barplot and rm_data_s is not None:
data_s = data_s.rolling(rm_data_s).mean()
elif not barplot and rs_data_s is not None:
data_s = data_s.resample(rs_data_s, label='left').mean()
rango_ts = data_s.index[0], data_s.index[-1]
date_ini, date_fin = [t.to_pydatetime() for t in rango_ts]
if data_s is not None and not data_s.empty:
lw, alpha = 1.5, 1.
ax.grid(b=True, which='major')
data_s = data_s.fillna(0)
if barplot:
div = .5
ylim = (0, np.ceil((data_s.max() + div) // div) * div)
ax.bar(data_s.index, data_s, width=1 / 28, edgecolor=color, color=[1, 1, 1, .5], linewidth=lw)
ax.xaxis.set_major_locator(mpd.HourLocator((0, 12)))
# ax.set_xticks([])
else:
if data_s.name == 'power':
div = 500
ylim = (0, np.ceil((data_s.max() + div / 5) / div) * div)
else: # ldr
div = 100
ylim = (0, np.ceil((data_s.max() + div / 2) // div) * div)
data_s = data_s.fillna(0)
ax.plot(data_s.index, data_s, color=color, linewidth=lw, alpha=alpha)
ax.fill_between(data_s.index, data_s, color=color, alpha=alpha / 2)
ax.xaxis.set_major_locator(mpd.HourLocator((0, 12)))
ax.xaxis.set_minor_locator(mpd.HourLocator(interval=1))
else:
ylim = 0, 100
ax.annotate('NO DATA!', xy=(.35, .3), xycoords='axes fraction',
va='center', ha='center', color=(.9, .9, .9), fontsize=25)
_adjust_tile_limits(data_s.name, ylim, date_ini, date_fin, ax)
return fig, ax
@timeit('gen_svg_tiles')
def gen_svg_tiles(path_dest, catalog, last_hours=(72, 48, 24)):
total_hours = last_hours[0]
last_data, last_data_c = catalog.get(last_hours=total_hours, with_summary_data=True)
if last_data is not None:
ahora = dt.datetime.now().replace(second=0, microsecond=0)
xlim = mpd.date2num(ahora - dt.timedelta(hours=total_hours)), mpd.date2num(ahora)
delta = xlim[1] - xlim[0]
fig, ax = None, None
for data_s, plot_bar in zip([last_data.power, last_data.ldr, last_data_c.kWh],
[False, False, True]):
if ax is not None:
plt.cla()
fig.set_figwidth(_tile_figsize()[0])
fig, ax = plot_tile_last_24h(data_s, rs_data_s='5min', barplot=plot_bar, ax=ax, fig=fig) # , rm_data_s=300)
for lh in last_hours:
file = os.path.join(path_dest, 'tile_{}_{}_last_{}h.svg'.format('enerpi_data', data_s.name, lh))
ax.set_xlim((xlim[0] + delta * (1 - lh / total_hours), xlim[1]))
fig.set_figwidth(_tile_figsize(lh / total_hours)[0])
_write_fig_to_svg(fig, name_img=file)
if fig is not None:
plt.close(fig)
return True
else:
return False
gen_svg_tiles('/Users/uge/ENERPIDATA/PLOTS', catalog, last_hours=(96, 84, 72, 60, 48, 36, 24, 12))
Out[28]:
In [27]:
gen_svg_tiles('/Users/uge/ENERPIDATA/PLOTS', catalog, last_hours=(72, 48, 24))
Out[27]:
In [37]:
mpd.HourLocator?
#mpd.DayLocator?
In [52]:
with pd.HDFStore('/Users/uge/Dropbox/PYTHON/PYPROJECTS/enerpi/enerpiweb/static/debug.h5', mode='r') as st:
data = st['debug']
data['T'] = data.index.map(lambda x: x.time().strftime('%H:%M:%S'))
data.head()
Out[52]:
In [124]:
from ipywidgets import HTML
df = data.reset_index(drop=True)[['T', 'Power (W)', 'LDR (%)', 'nº samples']].head()
HTML(df.to_html(justify='center', index=False, bold_rows=True, notebook=True, show_dimensions=True))
In [159]:
new_html = (
df.style.caption
.format(percent)
.applymap(color_negative_red, subset=['col1', 'col2'])
.set_properties(**{'font-size': '9pt', 'font-family': 'Calibri'})
.bar(subset=['col4', 'col5'], color='lightblue')
.render()
)
In [169]:
#df.style.set_uuid('idunico-tablebuffer').render()
#df.style.set_table_attributes('class="table-responsive"').render()
template = Template("""
<style type="text/css" >
{% for s in table_styles %}
#T_{{uuid}} {{s.selector}} {
{% for p,val in s.props %}
{{p}}: {{val}};
{% endfor %}
}
{% endfor %}
{% for s in cellstyle %}
#T_{{uuid}}{{s.selector}} {
{% for p,val in s.props %}
{{p}}: {{val}};
{% endfor %}
}
{% endfor %}
</style>
<table id="Table_{{uuid}}" {{ table_attributes }}>
{% if caption %}
<caption>{{caption}}</caption>
{% endif %}
<thead>
{% for r in head %}
<tr>
{% for c in r %}
<{{c.type}} class="{{c.class}}">{{c.value}}
{% endfor %}
</tr>
{% endfor %}
</thead>
<tbody>
{% for r in body %}
<tr>
{% for c in r %}
<{{c.type}} id="T_{{uuid}}_{{c.id}}" class="{{c.class}}">
{{ c.display_value }}
{% endfor %}
</tr>
{% endfor %}
</tbody>
</table>
""")
Out[169]:
In [171]:
pd.Timestamp('2016-08-20 20:31:32.854454').replace(microsecond=0)
Out[171]:
In [ ]:
In [ ]:
In [ ]:
In [158]:
from enerpi import BASE_PATH
paleta = pd.read_csv(os.path.join(BASE_PATH, 'rsc', 'paleta_power_w.csv')
).set_index('Unnamed: 0')['0'].str[1:-1].str.split(', ').apply(lambda x: [float(i) for i in x])
def aplica_paleta(serie):
return ['background-color: rgba({}, {}, {}, .7); color: #fff'.format(
*map(lambda x: int(255 * x), paleta.loc[:v].iloc[-1])) for v in serie]
#df.style
#paleta
#valores_w = [100, 150, 225, 300, 500, 1000, 1500, 2000, 4000, 6000]
#colores = [paleta.loc[:v].iloc[-1] for v in valores_w]
#sns.palplot(colores)
#plt.show()
html_styled = (df.style
.apply(aplica_paleta, subset=['Power (W)'])
.format({'LDR (%)': lambda x: "{:.1f} %".format(x),
'Power (W)': lambda x: "<strong>{}</strong> W".format(x)})
.bar(subset=['LDR (%)'], color='lightblue')
.render()
)
print_blue(html_styled)
HTML(html_styled)
In [141]:
(df.style
.apply(aplica_paleta, subset=['Power (W)'])
#.format({'LDR (%)': lambda x: "{:.1f} %".format(x), 'Power (W)': lambda x: "<strong>{}</strong> W".format(x)})
.bar(subset=['LDR (%)'], color='yellow')
.set_properties(**{'font-size': '12pt', 'font-family': 'Calibri'})
)
Out[141]:
In [ ]:
In [157]:
print(pd.formats.style.Styler.template.render())
pd.formats.style.Styler.use?
In [ ]:
In [27]:
# Lectura de nginx.conf
import re
raw = '''server {
listen 80;
server_name localhost;
charset utf-8;
client_max_body_size 75M;
# EnerWeb
location = /enerweb { rewrite ^ /enerweb/; }
# location /enerweb { try_files $uri @enerweb; }
# location @enerweb {
location /enerweb/ {
include uwsgi_params;
uwsgi_param /home/pi/PYTHON/enerweb/enerwebapp/enerweb.wsgi.py /enerweb;
uwsgi_pass unix:/tmp/enerweb.sock;
uwsgi_read_timeout 300;
}
# EnerpiWeb
location = /enerpi { rewrite ^ /enerpi/; }
location /enerpi/ {
include uwsgi_params;
uwsgi_param /home/pi/PYTHON/enerpi/enerpiweb/__main__.py /enerpi;
uwsgi_pass unix:/tmp/enerpiweb.sock;
uwsgi_read_timeout 300;
}
# MotionEye
location = /cams { rewrite ^ /cams/; }
location /cams/ {
proxy_pass http://127.0.0.1:8765/;
proxy_read_timeout 120s;
#access_log off;
}
# Home Assistant (HASS)
location = /hass { rewrite ^ /hass/; }
location /hass/ {
proxy_pass http://127.0.0.1:8123/;
proxy_read_timeout 120s;
#access_log off;
}
}'''
rg_servers = re.compile('server {(.*)}', flags=re.DOTALL | re.MULTILINE)
rg_lines = re.compile('(\n+?\s+)', flags=re.DOTALL | re.MULTILINE)
print(rg_servers.findall(raw)[0])
nginx = pd.DataFrame(rg_lines.split(rg_servers.findall(raw)[0])[::2], columns=['msg'])
nginx.tail()
Out[27]:
In [33]:
nginx['comment'] = nginx['msg'].str.startswith('#')
nginx['open'] = (~nginx['comment'] & nginx['msg'].str.contains('\{')).cumsum()
nginx['close'] = (~nginx['comment'] & nginx['msg'].str.contains('\}')).cumsum()
nginx
Out[33]:
In [109]:
import configparser
config = configparser.RawConfigParser()
config.read('/Users/uge/Dropbox/PYTHON/PYPROJECTS/enerpi/enerpi/config_enerpi.ini')
{s: dict(config[s]) for s in config.sections()}
#
#parser.read_file(p_conf)
#print(open(p_conf, 'r').readlines())
Out[109]:
In [115]:
list(config.keys())
Out[115]:
In [116]:
config.get('ENERPI_DATA', 'LOGGING_LEVEL', fallback=4)
Out[116]:
In [127]:
pd.read_hdf('/Users/uge/ENERPIDATA/CURRENT_MONTH/DATA_2016_08_DAY_15.h5', '/hours')
Out[127]:
In [75]:
s_b = parser['broadcast']
dict(s_b.items())
Out[75]:
In [119]:
config['BROADCAST']['udp_port']
Out[119]:
In [113]:
s_b.getint('UDP_PORT', fallback=57775)
Out[113]:
In [117]:
dict(config['RGBLED'].getint())
In [99]:
from ipywidgets import HTML
last = last.dropna(how='all', axis=1)
HTML(last.to_html(columns=['tipo', 'msg'], classes=['table', 'table-responsive', 'table-hover'], notebook=True))
In [121]:
dict(config['ENERPI_SAMPLER'])
Out[121]:
In [ ]:
In [ ]: