In [1]:
%matplotlib inline
%load_ext autoreload
%autoreload 2
import pandas as pd
pd.options.display.mpl_style = 'default'
import numpy as np
from abtest_util import SimStream, DonationProb, EmpiricalDonationProb
from bayesian_abtest import CostABTest
from nh_abtest import NHABTest, samples_per_branch_calculator
from abstract_abtest import expected_results, expected_results_by_lift
In [2]:
#Define Ground Truth Click Through Rates
# DonationProb class
p_B = DonationProb(0.19)
p_A = DonationProb(0.20)
# Estimate Control CTR From Historical Data
n = 10000
hist_data_B = SimStream(p_B).get_next_records(n)
p_hat = EmpiricalDonationProb(hist_data_B, p_B.values)
ci = p_hat.p_donate_ci()
print "CI over control:", ci
interval = round(1.0/ci[0])*50
print "Evalaution Interval", interval
#interest in lift vlaues:
lifts = [-0.20, -0.10, -0.05, -0.025, -0.015, 0.015, 0.025, 0.05, 0.10, 0.20]
In [ ]:
# Set Up Cost AB Test
cost = 0.001
max_run = float('inf') #this one is so clean it doesnt need a max_run arg
iters = 200
expected_results_by_lift(CostABTest,[None, None, interval, max_run, cost], iters, p_hat, lifts)
In [5]:
# Set Up NH AB Test
mde = 0.05
alpha = 0.05
power = 0.95
max_run = samples_per_branch_calculator(p_hat, mde, alpha, power)
print max_run
iters = 20
expected_results_by_lift(NHABTest,[None, None, interval, max_run, alpha], iters, p_hat, lifts)
Out[5]: