In [1]:

    

import numpy as np
import pandas as pd
import math
import sys


    

In [2]:

    

sys.version


    

    
Out[2]:





'3.6.0 |Continuum Analytics, Inc.| (default, Dec 23 2016, 12:22:00) \n[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]'

In [3]:

    

nan1 = float('nan')
nan2 = math.nan
nan3 = np.nan
nan1, nan2, nan3


    

    
Out[3]:





(nan, nan, nan)

In [4]:

    

pd.nan


    

    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-4-d7b1bc79a23a> in <module>()
----> 1 pd.nan

AttributeError: module 'pandas' has no attribute 'nan'

In [5]:

    

list(map(id, (nan1, nan2, nan3)))


    

    
Out[5]:





[140147194452128, 140147976447008, 140147753612416]

In [6]:

    

list(map(math.isnan, (nan1, nan2, nan3)))


    

    
Out[6]:





[True, True, True]

In [7]:

    

list(map(np.isnan, (nan1, nan2, nan3)))


    

    
Out[7]:





[True, True, True]

In [8]:

    

nan1 == nan1


    

    
Out[8]:





False

In [9]:

    

a = pd.Series([0., 0., 0., 1.])
a


    

    
Out[9]:





0    0.0
1    0.0
2    0.0
3    1.0
dtype: float64

In [10]:

    

b = pd.Series([0., 0., float('nan'), 1.])
b


    

    
Out[10]:





0    0.0
1    0.0
2    NaN
3    1.0
dtype: float64

In [11]:

    

c = pd.Series([float('nan'), float('nan'), float('nan'), float('nan'), ])
c


    

    
Out[11]:





0   NaN
1   NaN
2   NaN
3   NaN
dtype: float64

In [12]:

    

def show_dir(x):
    for s in ('nan', 'mean', 'tile'):
        for name in dir(x):
            if s in name.lower():
                print(name)
        print()


    

In [13]:

    

show_dir(a)


    

    




hasnans

mean

_check_percentile
quantile

In [14]:

    

show_dir(pd)


    

    




expanding_mean
rolling_mean

expanding_quantile
rolling_quantile

In [15]:

    

np.NAN is np.NaN


    

    
Out[15]:





True

In [16]:

    

a.mean()


    

    
Out[16]:





0.25

In [17]:

    

a.nanmean()


    

    




---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-17-b829d53aa1df> in <module>()
----> 1 a.nanmean()

/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/pandas/core/generic.py in __getattr__(self, name)
   2968             if name in self._info_axis:
   2969                 return self[name]
-> 2970             return object.__getattribute__(self, name)
   2971 
   2972     def __setattr__(self, name, value):

AttributeError: 'Series' object has no attribute 'nanmean'

In [18]:

    

np.mean(a), np.nanmean(a)


    

    
Out[18]:





(0.25, 0.25)

In [19]:

    

b.mean()


    

    
Out[19]:





0.33333333333333331

In [20]:

    

np.mean(b), np.nanmean(b)


    

    
Out[20]:





(0.33333333333333331, 0.33333333333333331)

In [21]:

    

c.mean()


    

    
Out[21]:





nan

In [22]:

    

np.mean(c)


    

    
Out[22]:





nan

In [23]:

    

np.nanmean(c)


    

    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/ipykernel/__main__.py:1: RuntimeWarning: Mean of empty slice
  if __name__ == '__main__':






    
Out[23]:





nan

In [24]:

    

a.sum(), b.sum(), c.sum()


    

    
Out[24]:





(1.0, 1.0, nan)

In [25]:

    

a.quantile(.1), b.quantile(.1), c.quantile(.1)


    

    
Out[25]:





(0.0, 0.0, nan)

In [26]:

    

>>> df = pd.DataFrame([1, 2, 3, np.nan], columns = ['x'])
>>> print(df.x.quantile(0.5))


    

    




2.0

In [27]:

    

np.sum(a), np.sum(b), np.sum(c)


    

    
Out[27]:





(1.0, 1.0, nan)

In [28]:

    

np.nansum(a), np.nansum(b), np.nansum(c)


    

    
Out[28]:





(1.0, 1.0, 0.0)

In [29]:

    

help(np.percentile)


    

    




Help on function percentile in module numpy.lib.function_base:

percentile(a, q, axis=None, out=None, overwrite_input=False, interpolation='linear', keepdims=False)
    Compute the qth percentile of the data along the specified axis.
    
    Returns the qth percentile(s) of the array elements.
    
    Parameters
    ----------
    a : array_like
        Input array or object that can be converted to an array.
    q : float in range of [0,100] (or sequence of floats)
        Percentile to compute, which must be between 0 and 100 inclusive.
    axis : {int, sequence of int, None}, optional
        Axis or axes along which the percentiles are computed. The
        default is to compute the percentile(s) along a flattened
        version of the array. A sequence of axes is supported since
        version 1.9.0.
    out : ndarray, optional
        Alternative output array in which to place the result. It must
        have the same shape and buffer length as the expected output,
        but the type (of the output) will be cast if necessary.
    overwrite_input : bool, optional
        If True, then allow use of memory of input array `a`
        calculations. The input array will be modified by the call to
        `percentile`. This will save memory when you do not need to
        preserve the contents of the input array. In this case you
        should not make any assumptions about the contents of the input
        `a` after this function completes -- treat it as undefined.
        Default is False. If `a` is not already an array, this parameter
        will have no effect as `a` will be converted to an array
        internally regardless of the value of this parameter.
    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
        This optional parameter specifies the interpolation method to
        use when the desired quantile lies between two data points
        ``i < j``:
            * linear: ``i + (j - i) * fraction``, where ``fraction``
              is the fractional part of the index surrounded by ``i``
              and ``j``.
            * lower: ``i``.
            * higher: ``j``.
            * nearest: ``i`` or ``j``, whichever is nearest.
            * midpoint: ``(i + j) / 2``.
    
        .. versionadded:: 1.9.0
    keepdims : bool, optional
        If this is set to True, the axes which are reduced are left in
        the result as dimensions with size one. With this option, the
        result will broadcast correctly against the original array `a`.
    
        .. versionadded:: 1.9.0
    
    Returns
    -------
    percentile : scalar or ndarray
        If `q` is a single percentile and `axis=None`, then the result
        is a scalar. If multiple percentiles are given, first axis of
        the result corresponds to the percentiles. The other axes are
        the axes that remain after the reduction of `a`. If the input
        contains integers or floats smaller than ``float64``, the output
        data-type is ``float64``. Otherwise, the output data-type is the
        same as that of the input. If `out` is specified, that array is
        returned instead.
    
    See Also
    --------
    mean, median, nanpercentile
    
    Notes
    -----
    Given a vector ``V`` of length ``N``, the ``q``-th percentile of
    ``V`` is the value ``q/100`` of the way from the mimumum to the
    maximum in in a sorted copy of ``V``. The values and distances of
    the two nearest neighbors as well as the `interpolation` parameter
    will determine the percentile if the normalized ranking does not
    match the location of ``q`` exactly. This function is the same as
    the median if ``q=50``, the same as the minimum if ``q=0`` and the
    same as the maximum if ``q=100``.
    
    Examples
    --------
    >>> a = np.array([[10, 7, 4], [3, 2, 1]])
    >>> a
    array([[10,  7,  4],
           [ 3,  2,  1]])
    >>> np.percentile(a, 50)
    3.5
    >>> np.percentile(a, 50, axis=0)
    array([[ 6.5,  4.5,  2.5]])
    >>> np.percentile(a, 50, axis=1)
    array([ 7.,  2.])
    >>> np.percentile(a, 50, axis=1, keepdims=True)
    array([[ 7.],
           [ 2.]])
    
    >>> m = np.percentile(a, 50, axis=0)
    >>> out = np.zeros_like(m)
    >>> np.percentile(a, 50, axis=0, out=out)
    array([[ 6.5,  4.5,  2.5]])
    >>> m
    array([[ 6.5,  4.5,  2.5]])
    
    >>> b = a.copy()
    >>> np.percentile(b, 50, axis=1, overwrite_input=True)
    array([ 7.,  2.])
    >>> assert not np.all(a == b)

In [30]:

    

for x in (a, b):
    for f in (np.percentile, np.nanpercentile):
        print(f'for {f}({x})')
        y = f(x, q=10)
        print(y)


    

    




for <function percentile at 0x7f769806d510>(0    0.0
1    0.0
2    0.0
3    1.0
dtype: float64)
0.0
for <function nanpercentile at 0x7f769807cd08>(0    0.0
1    0.0
2    0.0
3    1.0
dtype: float64)
0.0
for <function percentile at 0x7f769806d510>(0    0.0
1    0.0
2    NaN
3    1.0
dtype: float64)
nan
for <function nanpercentile at 0x7f769807cd08>(0    0.0
1    0.0
2    NaN
3    1.0
dtype: float64)
0.0






    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/numpy/lib/function_base.py:4116: RuntimeWarning: Invalid value encountered in percentile
  interpolation=interpolation)

In [31]:

    

q = 10


    

In [32]:

    

np.percentile(a, q)


    

    
Out[32]:





0.0

In [33]:

    

np.percentile(b, q)


    

    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/numpy/lib/function_base.py:4116: RuntimeWarning: Invalid value encountered in percentile
  interpolation=interpolation)






    
Out[33]:





nan

In [34]:

    

np.percentile(c, q)


    

    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/numpy/lib/function_base.py:4116: RuntimeWarning: Invalid value encountered in percentile
  interpolation=interpolation)






    
Out[34]:





nan

In [35]:

    

np.nanpercentile(a, q)


    

    
Out[35]:





0.0

In [36]:

    

np.nanpercentile(b, q)


    

    
Out[36]:





0.0

In [37]:

    

np.nanpercentile(c, q)


    

    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/numpy/lib/function_base.py:3858: RuntimeWarning: All-NaN slice encountered
  r = func(a, **kwargs)






    
Out[37]:





nan

In [38]:

    

def my_nanpercentile(*args, **kwargs):
    try:
        y = np.nanpercentile(*args, **kwargs)
    except RuntimeWarning:
        y = np.nan
    return y


    

In [39]:

    

my_nanpercentile(a, q)


    

    
Out[39]:





0.0

In [40]:

    

my_nanpercentile(b, q)


    

    
Out[40]:





0.0

In [41]:

    

my_nanpercentile(c, q)


    

    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/numpy/lib/function_base.py:3858: RuntimeWarning: All-NaN slice encountered
  r = func(a, **kwargs)






    
Out[41]:





nan

In [42]:

    

def my2_nanpercentile(*args, **kwargs):
    x = args[0]
    only_nans = all(map(math.isnan, x))
    if only_nans:
        y = np.nan
    else:
        y = np.nanpercentile(*args, **kwargs)
    return y


    

In [43]:

    

my2_nanpercentile(a, q)


    

    
Out[43]:





0.0

In [44]:

    

my2_nanpercentile(b, q)


    

    
Out[44]:





0.0

In [45]:

    

my2_nanpercentile(c, q)


    

    
Out[45]:





nan

In [46]:

    

np.mean(a)


    

    
Out[46]:





0.25

In [47]:

    

np.mean(b)


    

    
Out[47]:





0.33333333333333331

In [48]:

    

np.mean(c)


    

    
Out[48]:





nan

In [49]:

    

np.nanmean(a)


    

    
Out[49]:





0.25

In [50]:

    

np.nanmean(b)


    

    
Out[50]:





0.33333333333333331

In [51]:

    

np.nanmean(c)


    

    




/home/jep/anaconda3/envs/jupy/lib/python3.6/site-packages/ipykernel/__main__.py:1: RuntimeWarning: Mean of empty slice
  if __name__ == '__main__':






    
Out[51]:





nan

In [52]:

    

np.sum(a)


    

    
Out[52]:





1.0

In [53]:

    

np.sum(b)


    

    
Out[53]:





1.0

In [54]:

    

np.sum(c)


    

    
Out[54]:





nan

In [55]:

    

np.nansum(a)


    

    
Out[55]:





1.0

In [56]:

    

np.nansum(b)


    

    
Out[56]:





1.0

In [57]:

    

np.nansum(c)


    

    
Out[57]:





0.0