In [1]:
import sys
sys.path.insert(0, '../src/')
sys.path.insert(0, '../')
import django
django.setup()
In [2]:
from importlib import reload
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import ipywidgets as widgets
from IPython.display import display, clear_output
from ipywidgets import interact, interact_manual, Layout
In [3]:
import luminol.anomaly_detector as lad
from apps.utils.time import UTC_P0100
import apps.ad.anomaly_detection as ad
from apps.mc.api.util import get_topics, get_property, get_time_slots, get_features_of_interest, get_aggregating_process, get_observation_getter
from psycopg2.extras import DateTimeTZRange
In [4]:
phenomenon_date_from = pd.to_datetime("2019-01-01")
phenomenon_date_to = pd.to_datetime("2019-03-30")
In [5]:
d_topic = get_topics()[0]
d_prop = get_property(d_topic)[0]
d_feature = get_features_of_interest(d_topic, d_prop)[0]
d_time_slot = get_time_slots(d_topic)[0]
In [6]:
def detect_anomalies(
phenomenon_date_from = phenomenon_date_from,
phenomenon_date_to = phenomenon_date_to,
detector_method='bitmap_mod',
use_baseline=True,
shift=True,
extend_range=True,
detector_params={
"precision": 6,
"lag_window_size": 96,
"future_window_size": 96,
"chunk_size": 2
},
topic = d_topic,
prop = d_prop,
feature = d_feature,
time_slot = d_time_slot
):
pt_range_z = DateTimeTZRange(
pd.to_datetime(phenomenon_date_from).replace(tzinfo=UTC_P0100),
pd.to_datetime(phenomenon_date_to).replace(tzinfo=UTC_P0100)
)
get_func, feature_time_slots = get_observation_getter(
topic,
prop,
time_slot,
feature,
pt_range_z
)
anoms = ad.get_timeseries(
phenomenon_time_range=pt_range_z,
num_time_slots=len(feature_time_slots),
get_observations=get_func,
detector_method=detector_method,
detector_params=detector_params,
shift=shift,
use_baseline=use_baseline,
extend_range=extend_range,
)
anoms["feature_time_slots"] = feature_time_slots
return anoms
In [7]:
def highlight(indices, alpha, color, ax):
i=0
while i<len(indices):
ax.axvspan(indices[i]-0.5, indices[i]+0.5, facecolor=color, edgecolor='none', alpha=alpha)
i+=1
In [8]:
colors = ['r', 'g', 'c', 'm', 'y', 'k']
results = []
plt.ioff()
def plot(detectors, hlt_detector):
results = detectors
plt.close()
fig, ax1 = plt.subplots(figsize=(20,7))
hs = pd.DataFrame({
'anomalies': detectors[hlt_detector]["property_anomaly_rates"]
})
if detectors[hlt_detector]["property_anomaly_percentiles"]:
perc = detectors[hlt_detector]["property_anomaly_percentiles"]
color = colors[list(detectors.keys()).index(hlt_detector)]
for p in perc.keys():
highlight(hs[hs['anomalies'] > perc[p]].index, p*0.0025, color, ax1)
first_result = detectors[list(detectors.keys())[0]]
lns = []
if first_result["property_values"]:
ts = pd.DataFrame({
'values': [float(n) if n is not None else n for n in first_result["property_values"]]
}, index=[n.lower.strftime("%-d.%-m.%Y") for n in first_result["feature_time_slots"]])
values_line = ts['values'].plot.line(ax=ax1, color='b')
ax1.set_ylabel('values', color='b')
ax1.tick_params('y', colors='b')
lns.append(values_line.get_lines()[0])
for i in range(len(detectors.keys())):
detector = list(detectors.keys())[i]
color = colors[i]
anomalies = detectors[detector]["property_anomaly_rates"]
if anomalies:
ts = pd.DataFrame({
'anomalies': anomalies
})
ax2 = ax1.twinx()
anomalies_line = ts['anomalies'].plot.line(ax=ax2, color=color, label=detector)
lns.append(anomalies_line.get_lines()[0])
ax2.tick_params('y', colors=color)
if lns:
labs = [ln.get_label() for ln in lns]
ax1.legend(lns, labs, loc=1)
# baseName = f"{baserange.lower.date()}..{baserange.upper.date()}"
# if first_result['phenomenon_time_range']:
# rangeName = f"{first_result['phenomenon_time_range'].lower.date()}..{first_result['phenomenon_time_range'].upper.date()}"
# plt.savefig(f"graphs/{baseName}_{rangeName}_window-{str(window_size)}_prec-{str(detector_params['precision'])}.png", format="png")
# plt.savefig(f"graphs/{rangeName}_window-{str(window_size)}_prec-{str(detector_params['precision'])}.png", format="png")
return fig
In [9]:
detectors = {
"Bitmap mod": "bitmap_mod",
"Bitmap diminishing": "bitmap_diminishing",
"Bitmap diminishing baseline": "bitmap_diminishing_bl",
"Bitmap mod shift": "bitmap_mod_shift",
"LinkedIn bitmap": "bitmap_detector",
"Default": "default_detector",
"Derivative": "derivative_detector",
"Exponential average": "exp_avg_detector",
# "Absolute threshold": "absolute_threshold",
# "Diff Percent": "diff_percent_threshold",
# "Sign test": "sign_test",
}
In [10]:
def val_bitmap_mod(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "bitmap_mod", shift=False, feature=feature, prop=prop, time_slot=time_slot, detector_params=detector_params)
def val_bitmap_diminishing(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "bitmap_diminishing", shift=False, use_baseline=False, feature=feature, prop=prop, time_slot=time_slot, detector_params=detector_params)
def val_bitmap_diminishing_bl(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "bitmap_diminishing", shift=False, feature=feature, prop=prop, time_slot=time_slot, detector_params=detector_params)
def val_bitmap_mod_shift(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "bitmap_mod_shift", feature=feature, prop=prop, time_slot=time_slot, detector_params=detector_params)
def val_bitmap_detector(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "bitmap_detector", shift=False, use_baseline=False, feature=feature, prop=prop, time_slot=time_slot, detector_params=detector_params)
def val_default_detector(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "default_detector", feature=feature, prop=prop, time_slot=time_slot, detector_params={}, use_baseline=False, extend_range=False, shift=False)
def val_derivative_detector(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "derivative_detector", feature=feature, prop=prop, time_slot=time_slot, detector_params={}, use_baseline=False, extend_range=False, shift=False)
def val_exp_avg_detector(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(start_date, end_date, "exp_avg_detector", feature=feature, prop=prop, time_slot=time_slot, detector_params={}, use_baseline=False, extend_range=False, shift=False)
def val_absolute_threshold(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(t_from, t_to, "absolute_threshold", feature=feature, prop=prop, time_slot=time_slot, detector_params={}, use_baseline=False, extend_range=False, shift=False)
def val_diff_percent_threshold(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(t_from, t_to, "diff_percent_threshold", feature=feature, prop=prop, time_slot=time_slot, detector_params={}, use_baseline=False, extend_range=False, shift=False)
def val_sign_test(feature, prop, time_slot, start_date, end_date, detector_params):
return detect_anomalies(t_from, t_to, "sign_test", feature=feature, prop=prop, time_slot=time_slot, detector_params={}, use_baseline=False, extend_range=False, shift=False)
def plot_anomalies(feature, prop, time_slot, start_date, end_date, precision, window_size, chunk_size, hlt_detector, bitmap_mod, bitmap_diminishing, bitmap_diminishing_bl, bitmap_mod_shift, bitmap_detector, default_detector, derivative_detector, exp_avg_detector):
args = [feature, prop, time_slot, start_date, end_date, {
"precision": precision,
"lag_window_size": window_size,
"future_window_size": window_size,
"chunk_size": chunk_size
}]
anomalies = {}
if bitmap_mod:
anomalies["bitmap_mod"] = val_bitmap_mod(*args)
if bitmap_diminishing:
anomalies["bitmap_diminishing"] = val_bitmap_diminishing(*args)
if bitmap_diminishing_bl:
anomalies["bitmap_diminishing_bl"] = val_bitmap_diminishing_bl(*args)
if bitmap_mod_shift:
anomalies["bitmap_mod_shift"] = val_bitmap_mod_shift(*args)
if bitmap_detector:
anomalies["bitmap_detector"] = val_bitmap_detector(*args)
if default_detector:
anomalies["default_detector"] = val_default_detector(*args)
if derivative_detector:
anomalies["derivative_detector"] = val_derivative_detector(*args)
if exp_avg_detector:
anomalies["exp_avg_detector"] = val_exp_avg_detector(*args)
# if absolute_threshold:
# anomalies["absolute_threshold"] = val_absolute_threshold(*args)
# if diff_percent_threshold:
# anomalies["diff_percent_threshold"] = val_diff_percent_threshold(*args)
# if sign_test:
# anomalies["sign_test"] = val_sign_test(*args)
if len(anomalies.keys()) > 1:
i = 0
while hlt_detector not in anomalies.keys():
hlt_detector = detectors[i]
i += 1
return plot(anomalies, hlt_detector)
In [11]:
t_from = "2019-01-01"
t_to = "2019-03-30"
def hlt_detectors():
d = {}
if bitmap_mod_widget.value: d["Bitmap mod"] = "bitmap_mod"
if bitmap_diminishing_widget.value: d["Bitmap diminishing"] = "bitmap_diminishing"
if bitmap_diminishing_bl_widget.value: d["Bitmap diminishing baseline"] = "bitmap_diminishing_bl"
if bitmap_mod_shift_widget.value: d["Bitmap mod shift"] = "bitmap_mod_shift"
if bitmap_detector_widget.value: d["LinkedIn bitmap"] = "bitmap_detector"
if default_detector_widget.value: d["Default"] = "default_detector"
if derivative_detector_widget.value: d["Derivative"] = "derivative_detector"
if exp_avg_detector_widget.value: d["Exponential average"] = "exp_avg_detector"
# if absolute_threshold_widget.value: d["Absolute threshold"] = "absolute_threshold"
# if diff_percent_threshold_widget.value: d["Diff Percent"] = "diff_percent_threshold"
# if sign_test_widget.value: d["Sign test"] = "sign_test"
return d
def update_hlt_detectors(*args):
hlt_detector_widget.options = hlt_detectors()
def update_property_widget(*args):
property_widget.options = get_property(topic_widget.value)
def update_feature_widget(*args):
feature_widget.options = get_features_of_interest(topic_widget.value, property_widget.value)
def update_time_slots_widget(*args):
time_slots_widget.options = get_time_slots(topic_widget.value)
In [12]:
bitmap_mod_widget = widgets.Checkbox(value=False,description="Bitmap mod")
bitmap_mod_widget.observe(update_hlt_detectors, "value")
bitmap_diminishing_widget = widgets.Checkbox(value=True,description="Bitmap diminishing")
bitmap_diminishing_widget.observe(update_hlt_detectors, "value")
bitmap_diminishing_bl_widget = widgets.Checkbox(value=True,description="Bitmap diminishing baseline")
bitmap_diminishing_bl_widget.observe(update_hlt_detectors, "value")
bitmap_mod_shift_widget = widgets.Checkbox(value=False,description="Bitmap mod shift")
bitmap_mod_shift_widget.observe(update_hlt_detectors, "value")
bitmap_detector_widget = widgets.Checkbox(value=False,description="LinkedIn bitmap")
bitmap_detector_widget.observe(update_hlt_detectors, "value")
default_detector_widget = widgets.Checkbox(value=False,description="Default")
default_detector_widget.observe(update_hlt_detectors, "value")
derivative_detector_widget = widgets.Checkbox(value=False,description="Derivative")
derivative_detector_widget.observe(update_hlt_detectors, "value")
exp_avg_detector_widget = widgets.Checkbox(value=False,description="Exponential average")
exp_avg_detector_widget.observe(update_hlt_detectors, "value")
# absolute_threshold_widget = widgets.Checkbox(value=False,description="Absolute threshold")
# absolute_threshold_widget.observe(update_hlt_detectors, "value")
# diff_percent_threshold_widget = widgets.Checkbox(value=False,description="Diff Percent")
# diff_percent_threshold_widget.observe(update_hlt_detectors, "value")
# sign_test_widget = widgets.Checkbox(value=False,description="Sign test")
# sign_test_widget.observe(update_hlt_detectors, "value")
detector_widgets = widgets.HBox([
widgets.VBox([
bitmap_mod_widget,
bitmap_diminishing_widget,
bitmap_diminishing_bl_widget,
bitmap_detector_widget,
]),
widgets.VBox([
bitmap_mod_shift_widget,
default_detector_widget,
derivative_detector_widget,
exp_avg_detector_widget,
]),
# widgets.VBox([
# absolute_threshold_widget,
# diff_percent_threshold_widget,
# sign_test_widget
# ])
])
hlt_detector_widget = widgets.Dropdown(options=hlt_detectors(), value=detectors["Bitmap diminishing"], description="Highlight")
precision_widget = widgets.IntSlider(value=6, min=2, max=16, step=1, description="Precision")
window_size_widget = widgets.BoundedIntText(value=96, min=4, max=256, step=1, description="Window size")
chunk_size_widget = widgets.IntSlider(value=2, min=2, max=16, step=1, description="Chunk size")
topic_widget = widgets.Dropdown(options=get_topics(), description="Topic")
topic_widget.observe(update_property_widget, "value")
topic_widget.observe(update_feature_widget, "value")
topic_widget.observe(update_time_slots_widget, "value")
property_widget = widgets.Dropdown(options=get_property(topic_widget.value), description="Property")
property_widget.observe(update_feature_widget, "value")
feature_widget = widgets.Dropdown(options=get_features_of_interest(topic_widget.value, property_widget.value), description="Station")
time_slot_widget = widgets.Dropdown(options=get_time_slots(topic_widget.value), description="Aggregate to")
start_date_widget = widgets.DatePicker(value=pd.to_datetime(t_from).date(), description="Start date")
end_date_widget = widgets.DatePicker(value=pd.to_datetime(t_to).date(), description="End date")
date_widgets = widgets.HBox([
start_date_widget,
end_date_widget
])
# ui = widgets.Tab(children=[
widget_accordion = widgets.Accordion(children=[
widgets.VBox([
widgets.HBox([topic_widget, property_widget]),
widgets.HBox([feature_widget, time_slot_widget]),
date_widgets
]),
widgets.VBox([
precision_widget,
window_size_widget,
chunk_size_widget
]),
widgets.VBox([
detector_widgets,
hlt_detector_widget
])
],
# layout=Layout(
# height='300px',
# display='flex',
# align_items='center',
# justify_content='center'
# )
)
widget_accordion.set_title(0, "General")
widget_accordion.set_title(1, "Detector parameters")
widget_accordion.set_title(2, "Detectors used")
widget_accordion.selected_index = 0
out = widgets.Output()
plot_button = widgets.Button(
description="Plot"
)
def click(b):
fig = plot_anomalies(
bitmap_mod=bitmap_mod_widget.value,
bitmap_diminishing=bitmap_diminishing_widget.value,
bitmap_diminishing_bl=bitmap_diminishing_bl_widget.value,
bitmap_mod_shift=bitmap_mod_shift_widget.value,
bitmap_detector=bitmap_detector_widget.value,
default_detector=default_detector_widget.value,
derivative_detector=derivative_detector_widget.value,
exp_avg_detector=exp_avg_detector_widget.value,
# absolute_threshold=absolute_threshold_widget.value,
# diff_percent_threshold=diff_percent_threshold_widget.value,
# sign_test=sign_test_widget.value,
feature=feature_widget.value,
prop=property_widget.value,
time_slot=time_slot_widget.value,
start_date=start_date_widget.value,
end_date=end_date_widget.value,
precision=precision_widget.value,
window_size=window_size_widget.value,
chunk_size=chunk_size_widget.value,
hlt_detector=hlt_detector_widget.value
)
with out:
clear_output(wait=True)
display(fig)
plot_button.on_click(click)
ui = widgets.VBox([
widget_accordion,
plot_button,
out
])
def init():
reload(ad)
reload(lad)
display(ui)
click(None)
print("Aggregating process:", get_aggregating_process(topic_widget.value, property_widget.value, feature_widget.value))
In [13]:
init()