In [1]:

    

# running an A/B test
from math import sqrt


def estimated_parameters(N, n):
    '''estimating the parameters of N Bernoulli trials
    with n successes'''
    p = n / N
    sigma = sqrt(p * (1 - p) / N)
    return p, sigma


def a_b_test_statistic(N_A, n_A, N_B, n_B):
    '''test the null hypothesis that pA and pB are the same'''
    p_A, sigma_A = estimated_parameters(N_A, n_A)
    p_B, sigma_B = estimated_parameters(N_B, n_B)
    return (p_B - p_A) / sqrt(sigma_A ** 2 + sigma_B ** 2)


# let's compare two different experiments
z = a_b_test_statistic(1000, 200, 1000, 180)
print(z)


    

    




-1.1403464899034472

In [4]:

    

# what are is the chance of such a large difference if the means were equal?
from math import erf


def normal_cdf(x, mu=0, sigma=1):
    '''We can use a normal CDF to determine if a random variable
    falls within some range.
    '''
    return (1 + erf((x - mu) / sqrt(2) / sigma)) / 2


normal_prob_below = normal_cdf


def normal_prob_above(lo, mu=0, sigma=1):
    return 1 - normal_cdf(lo, mu, sigma)


def two_sided_p_value(x, mu=0, sigma=1):
    if x >= mu:
        return 2 * normal_prob_above(x, mu, sigma)
    else:
        return 2 * normal_prob_below(x, mu, sigma)


# what is the chance this is still the null hypothesis?
z = a_b_test_statistic(1000, 200, 1000, 180)
print(two_sided_p_value(z))


    

    




0.254141976542236

In [7]:

    

# What if our two tests had different results? Let's test the null hypothesis now.
z = a_b_test_statistic(1000, 200, 1000, 150)
print(z)
print(two_sided_p_value(z))


# Okay, 3%. This is starting to look like a trustworthy result.


    

    




-2.948839123097944
0.003189699706216853

In [8]:

    

# Bayesian Inference


    

In [ ]: