NumPy Basics: Arrays and Vectorized Computation



In [9]:

    
%matplotlib inline



In [10]:

    
from __future__ import division
from numpy.random import randn
import numpy as np
np.set_printoptions(precision=4, suppress=True)

The NumPy ndarray: a multidimensional array object



In [11]:

    
data = randn(2, 3)



In [12]:

    
data









    Out[12]:





array([[-0.2926,  0.3563,  0.6034],
       [-0.5578,  0.5153,  1.0498]])



In [13]:

    
data * 10









    Out[13]:





array([[ -2.926 ,   3.5631,   6.0341],
       [ -5.5783,   5.1525,  10.4985]])



In [14]:

    
data + data









    Out[14]:





array([[-0.5852,  0.7126,  1.2068],
       [-1.1157,  1.0305,  2.0997]])



In [15]:

    
data.shape









    Out[15]:





(2, 3)



In [16]:

    
data.dtype









    Out[16]:





dtype('float64')

Creating ndarrays



In [17]:

    
data1 = [6, 7.5, 8, 0, 1]
arr1 = np.array(data1)
arr1









    Out[17]:





array([ 6. ,  7.5,  8. ,  0. ,  1. ])



In [18]:

    
data2 = [[1, 2, 3, 4], [5, 6, 7, 8]]
arr2 = np.array(data2)
arr2
arr2.ndim
arr2.shape









    Out[18]:





(2, 4)



In [19]:

    
arr1.dtype
arr2.dtype









    Out[19]:





dtype('int64')



In [20]:

    
np.zeros(10)
np.zeros((3, 6))
np.empty((2, 3, 2))









    Out[20]:





array([[[ 0.,  0.],
        [ 0.,  0.],
        [ 0.,  0.]],

       [[ 0.,  0.],
        [ 0.,  0.],
        [ 0.,  0.]]])



In [21]:

    
np.arange(15)









    Out[21]:





array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14])

Data Types for ndarrays



In [22]:

    
arr1 = np.array([1, 2, 3], dtype=np.float64)
arr2 = np.array([1, 2, 3], dtype=np.int32)
arr1.dtype
arr2.dtype









    Out[22]:





dtype('int32')



In [23]:

    
arr = np.array([1, 2, 3, 4, 5])
arr.dtype
float_arr = arr.astype(np.float64)
float_arr.dtype









    Out[23]:





dtype('float64')



In [24]:

    
arr = np.array([3.7, -1.2, -2.6, 0.5, 12.9, 10.1])
arr
arr.astype(np.int32)









    Out[24]:





array([ 3, -1, -2,  0, 12, 10], dtype=int32)



In [25]:

    
numeric_strings = np.array(['1.25', '-9.6', '42'], dtype=np.string_)
numeric_strings.astype(float)









    Out[25]:





array([  1.25,  -9.6 ,  42.  ])



In [26]:

    
int_array = np.arange(10)
calibers = np.array([.22, .270, .357, .380, .44, .50], dtype=np.float64)
int_array.astype(calibers.dtype)









    Out[26]:





array([ 0.,  1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9.])



In [27]:

    
empty_uint32 = np.empty(8, dtype='u4')
empty_uint32









    Out[27]:





array([         0, 1075314688,          0, 1075707904,          0,
       1075838976,          0, 1072693248], dtype=uint32)

Operations between arrays and scalars



In [28]:

    
arr = np.array([[1., 2., 3.], [4., 5., 6.]])



In [29]:

    
arr









    Out[29]:





array([[ 1.,  2.,  3.],
       [ 4.,  5.,  6.]])



In [30]:

    
arr * arr









    Out[30]:





array([[  1.,   4.,   9.],
       [ 16.,  25.,  36.]])



In [31]:

    
arr - arr









    Out[31]:





array([[ 0.,  0.,  0.],
       [ 0.,  0.,  0.]])



In [32]:

    
1 / arr









    Out[32]:





array([[ 1.    ,  0.5   ,  0.3333],
       [ 0.25  ,  0.2   ,  0.1667]])



In [33]:

    
arr ** 0.5









    Out[33]:





array([[ 1.    ,  1.4142,  1.7321],
       [ 2.    ,  2.2361,  2.4495]])

Basic indexing and slicing



In [34]:

    
arr = np.arange(10)
arr
arr[5]
arr[5:8]
arr[5:8] = 12
arr









    Out[34]:





array([ 0,  1,  2,  3,  4, 12, 12, 12,  8,  9])



In [35]:

    
arr_slice = arr[5:8]
arr_slice[1] = 12345
arr
arr_slice[:] = 64
arr









    Out[35]:





array([ 0,  1,  2,  3,  4, 64, 64, 64,  8,  9])



In [36]:

    
arr2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
arr2d[2]









    Out[36]:





array([7, 8, 9])



In [37]:

    
arr2d[0][2]
arr2d[0, 2]









    Out[37]:





3



In [38]:

    
arr3d = np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])
arr3d









    Out[38]:





array([[[ 1,  2,  3],
        [ 4,  5,  6]],

       [[ 7,  8,  9],
        [10, 11, 12]]])



In [39]:

    
arr3d[0]









    Out[39]:





array([[1, 2, 3],
       [4, 5, 6]])



In [40]:

    
old_values = arr3d[0].copy()
arr3d[0] = 42
arr3d
arr3d[0] = old_values
arr3d









    Out[40]:





array([[[ 1,  2,  3],
        [ 4,  5,  6]],

       [[ 7,  8,  9],
        [10, 11, 12]]])



In [41]:

    
arr3d[1, 0]









    Out[41]:





array([7, 8, 9])

Indexing with slices



In [42]:

    
arr[1:6]









    Out[42]:





array([ 1,  2,  3,  4, 64])



In [43]:

    
arr2d
arr2d[:2]









    Out[43]:





array([[1, 2, 3],
       [4, 5, 6]])



In [44]:

    
arr2d[:2, 1:]









    Out[44]:





array([[2, 3],
       [5, 6]])



In [45]:

    
arr2d[1, :2]
arr2d[2, :1]









    Out[45]:





array([7])



In [46]:

    
arr2d[:, :1]









    Out[46]:





array([[1],
       [4],
       [7]])



In [47]:

    
arr2d[:2, 1:] = 0
arr2d









    Out[47]:





array([[1, 0, 0],
       [4, 0, 0],
       [7, 8, 9]])

Boolean indexing



In [48]:

    
names = np.array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'])
names









    Out[48]:





array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'], 
      dtype='|S4')



In [49]:

    
data = randn(7, 4)
data









    Out[49]:





array([[ 0.1065,  0.7593,  0.6358,  1.8325],
       [ 0.7604, -0.0798, -0.0331,  0.4773],
       [ 0.0237,  0.5677, -1.1509,  0.3842],
       [ 0.2073,  0.2709,  0.4719,  0.1125],
       [-0.6838, -0.2528,  1.3652,  0.8838],
       [ 3.0824,  1.3709, -0.2734, -1.7973],
       [-2.4717, -0.8027,  0.4641,  2.2261]])



In [50]:

    
data.shape









    Out[50]:





(7, 4)



In [51]:

    
names == 'Bob'









    Out[51]:





array([ True, False, False,  True, False, False, False], dtype=bool)



In [52]:

    
data[names == 'Bob']









    Out[52]:





array([[ 0.1065,  0.7593,  0.6358,  1.8325],
       [ 0.2073,  0.2709,  0.4719,  0.1125]])



In [53]:

    
data[names == 'Bob', 2:]
data[names == 'Bob', 3]









    Out[53]:





array([ 1.8325,  0.1125])



In [54]:

    
names != 'Bob'
data[- (names == 'Bob')]









    



/Users/pmui/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/ipykernel/__main__.py:2: DeprecationWarning: numpy boolean negative, the `-` operator, is deprecated, use the `~` operator or the logical_not function instead.
  from ipykernel import kernelapp as app






    Out[54]:





array([[ 0.7604, -0.0798, -0.0331,  0.4773],
       [ 0.0237,  0.5677, -1.1509,  0.3842],
       [-0.6838, -0.2528,  1.3652,  0.8838],
       [ 3.0824,  1.3709, -0.2734, -1.7973],
       [-2.4717, -0.8027,  0.4641,  2.2261]])



In [55]:

    
mask = (names == 'Bob') | (names == 'Will')
mask
data[mask]









    Out[55]:





array([[ 0.1065,  0.7593,  0.6358,  1.8325],
       [ 0.0237,  0.5677, -1.1509,  0.3842],
       [ 0.2073,  0.2709,  0.4719,  0.1125],
       [-0.6838, -0.2528,  1.3652,  0.8838]])



In [56]:

    
data[data < 0] = 0
data









    Out[56]:





array([[ 0.1065,  0.7593,  0.6358,  1.8325],
       [ 0.7604,  0.    ,  0.    ,  0.4773],
       [ 0.0237,  0.5677,  0.    ,  0.3842],
       [ 0.2073,  0.2709,  0.4719,  0.1125],
       [ 0.    ,  0.    ,  1.3652,  0.8838],
       [ 3.0824,  1.3709,  0.    ,  0.    ],
       [ 0.    ,  0.    ,  0.4641,  2.2261]])



In [57]:

    
data[names != 'Joe'] = 7
data









    Out[57]:





array([[ 7.    ,  7.    ,  7.    ,  7.    ],
       [ 0.7604,  0.    ,  0.    ,  0.4773],
       [ 7.    ,  7.    ,  7.    ,  7.    ],
       [ 7.    ,  7.    ,  7.    ,  7.    ],
       [ 7.    ,  7.    ,  7.    ,  7.    ],
       [ 3.0824,  1.3709,  0.    ,  0.    ],
       [ 0.    ,  0.    ,  0.4641,  2.2261]])

Fancy indexing



In [58]:

    
arr = np.empty((8, 4))
for i in range(8):
    arr[i] = i
arr









    Out[58]:





array([[ 0.,  0.,  0.,  0.],
       [ 1.,  1.,  1.,  1.],
       [ 2.,  2.,  2.,  2.],
       [ 3.,  3.,  3.,  3.],
       [ 4.,  4.,  4.,  4.],
       [ 5.,  5.,  5.,  5.],
       [ 6.,  6.,  6.,  6.],
       [ 7.,  7.,  7.,  7.]])



In [59]:

    
arr[[4, 3, 0, 6]]









    Out[59]:





array([[ 4.,  4.,  4.,  4.],
       [ 3.,  3.,  3.,  3.],
       [ 0.,  0.,  0.,  0.],
       [ 6.,  6.,  6.,  6.]])



In [60]:

    
arr[[-3, -5, -7]]









    Out[60]:





array([[ 5.,  5.,  5.,  5.],
       [ 3.,  3.,  3.,  3.],
       [ 1.,  1.,  1.,  1.]])



In [61]:

    
# more on reshape in Chapter 12
arr = np.arange(32).reshape((8, 4))
arr
arr[[1, 5, 7, 2], [0, 3, 1, 2]]









    Out[61]:





array([ 4, 23, 29, 10])



In [62]:

    
arr[[1, 5, 7, 2]][:, [0, 3, 1, 2]]









    Out[62]:





array([[ 4,  7,  5,  6],
       [20, 23, 21, 22],
       [28, 31, 29, 30],
       [ 8, 11,  9, 10]])



In [63]:

    
arr[np.ix_([1, 5, 7, 2], [0, 3, 1, 2])]









    Out[63]:





array([[ 4,  7,  5,  6],
       [20, 23, 21, 22],
       [28, 31, 29, 30],
       [ 8, 11,  9, 10]])

Transposing arrays and swapping axes



In [64]:

    
arr = np.arange(15).reshape((3, 5))
arr
arr.T









    Out[64]:





array([[ 0,  5, 10],
       [ 1,  6, 11],
       [ 2,  7, 12],
       [ 3,  8, 13],
       [ 4,  9, 14]])



In [65]:

    
arr = np.random.randn(6, 3)
np.dot(arr.T, arr)









    Out[65]:





array([[  5.429 ,   2.4362,  -5.9145],
       [  2.4362,   8.4699,  -7.9449],
       [ -5.9145,  -7.9449,  10.8886]])



In [66]:

    
arr = np.arange(16).reshape((2, 2, 4))
arr
arr.transpose((1, 0, 2))









    Out[66]:





array([[[ 0,  1,  2,  3],
        [ 8,  9, 10, 11]],

       [[ 4,  5,  6,  7],
        [12, 13, 14, 15]]])



In [67]:

    
arr
arr.swapaxes(1, 2)









    Out[67]:





array([[[ 0,  4],
        [ 1,  5],
        [ 2,  6],
        [ 3,  7]],

       [[ 8, 12],
        [ 9, 13],
        [10, 14],
        [11, 15]]])

Universal Functions: Fast element-wise array functions



In [68]:

    
arr = np.arange(10)
np.sqrt(arr)
np.exp(arr)









    Out[68]:





array([    1.    ,     2.7183,     7.3891,    20.0855,    54.5982,
         148.4132,   403.4288,  1096.6332,  2980.958 ,  8103.0839])



In [69]:

    
x = randn(8)
y = randn(8)
x
y
np.maximum(x, y) # element-wise maximum









    Out[69]:





array([-0.5438,  0.9202,  0.2609, -0.0576,  1.5691, -0.0864,  2.862 ,
       -0.7016])



In [ ]:

    
arr = randn(7) * 5
np.modf(arr)

Data processing using arrays



In [113]:

    
points = np.arange(-5, 5, 0.01) # 1000 equally spaced points
xs, ys = np.meshgrid(points, points)
ys









    Out[113]:





array([[-5.  , -5.  , -5.  , ..., -5.  , -5.  , -5.  ],
       [-4.99, -4.99, -4.99, ..., -4.99, -4.99, -4.99],
       [-4.98, -4.98, -4.98, ..., -4.98, -4.98, -4.98],
       ..., 
       [ 4.97,  4.97,  4.97, ...,  4.97,  4.97,  4.97],
       [ 4.98,  4.98,  4.98, ...,  4.98,  4.98,  4.98],
       [ 4.99,  4.99,  4.99, ...,  4.99,  4.99,  4.99]])



In [114]:

    
from matplotlib.pyplot import imshow, title



In [115]:

    
import matplotlib.pyplot as plt
z = np.sqrt(xs ** 2 + ys ** 2)
z
plt.imshow(z, cmap=plt.cm.gray); plt.colorbar()
plt.title("Image plot of $\sqrt{x^2 + y^2}$ for a grid of values")









    Out[115]:





<matplotlib.text.Text at 0x11550c150>



In [116]:

    
plt.draw()









    





<matplotlib.figure.Figure at 0x1153ca7d0>

Expressing conditional logic as array operations



In [120]:

    
xarr = np.array([1.1, 1.2, 1.3, 1.4, 1.5])
yarr = np.array([2.1, 2.2, 2.3, 2.4, 2.5])
cond = np.array([True, False, True, True, False])



In [121]:

    
result = [(x if c else y)
          for x, y, c in zip(xarr, yarr, cond)]
result









    Out[121]:





[1.1000000000000001, 2.2000000000000002, 1.3, 1.3999999999999999, 2.5]



In [122]:

    
result = np.where(cond, xarr, yarr)
result









    Out[122]:





array([ 1.1,  2.2,  1.3,  1.4,  2.5])



In [123]:

    
arr = randn(4, 4)
arr
np.where(arr > 0, 2, -2)
np.where(arr > 0, 2, arr) # set only positive values to 2









    Out[123]:





array([[-1.307 ,  2.    ,  2.    ,  2.    ],
       [-1.0305, -0.2168,  2.    ,  2.    ],
       [ 2.    ,  2.    ,  2.    , -1.3704],
       [-0.544 , -0.7909,  2.    , -0.5366]])



In [ ]:

    
# Not to be executed

result = []
for i in range(n):
    if cond1[i] and cond2[i]:
        result.append(0)
    elif cond1[i]:
        result.append(1)
    elif cond2[i]:
        result.append(2)
    else:
        result.append(3)



In [ ]:

    
# Not to be executed

np.where(cond1 & cond2, 0,
         np.where(cond1, 1,
                  np.where(cond2, 2, 3)))



In [ ]:

    
# Not to be executed

result = 1 * cond1 + 2 * cond2 + 3 * -(cond1 | cond2)

Mathematical and statistical methods



In [125]:

    
arr = np.random.randn(5, 4) # normally-distributed data
arr.mean()
np.mean(arr)
arr.sum()









    Out[125]:





0.95466471284258958



In [126]:

    
arr.mean(axis=1)
arr.sum(0)









    Out[126]:





array([-0.8918,  2.6701, -1.0717,  0.248 ])



In [127]:

    
arr = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8]])
arr.cumsum(0)
arr.cumprod(1)









    Out[127]:





array([[  0,   0,   0],
       [  3,  12,  60],
       [  6,  42, 336]])

Methods for boolean arrays



In [128]:

    
arr = randn(100)
(arr > 0).sum() # Number of positive values









    Out[128]:





45



In [129]:

    
bools = np.array([False, False, True, False])
bools.any()
bools.all()









    Out[129]:





False

Sorting



In [130]:

    
arr = randn(8)
arr









    Out[130]:





array([-0.8032, -0.0669,  0.2084, -1.2895, -1.1235, -0.3234, -1.9909,
       -1.0247])



In [131]:

    
arr.sort()
arr









    Out[131]:





array([-1.9909, -1.2895, -1.1235, -1.0247, -0.8032, -0.3234, -0.0669,
        0.2084])



In [132]:

    
arr = randn(5, 3)
arr









    Out[132]:





array([[-1.473 , -0.8698,  1.2525],
       [ 1.2563, -1.2063, -0.89  ],
       [-0.6116, -1.0631,  1.6946],
       [-0.0862, -2.1832,  0.2069],
       [ 0.9488,  1.418 , -1.0427]])



In [133]:

    
arr.sort(1)
arr









    Out[133]:





array([[-1.473 , -0.8698,  1.2525],
       [-1.2063, -0.89  ,  1.2563],
       [-1.0631, -0.6116,  1.6946],
       [-2.1832, -0.0862,  0.2069],
       [-1.0427,  0.9488,  1.418 ]])



In [ ]:

    
large_arr = randn(1000)
large_arr.sort()
large_arr[int(0.05 * len(large_arr))] # 5% quantile

Unique and other set logic



In [ ]:

    
names = np.array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'])
np.unique(names)
ints = np.array([3, 3, 3, 2, 2, 1, 1, 4, 4])
np.unique(ints)



In [ ]:

    
sorted(set(names))



In [ ]:

    
values = np.array([6, 0, 0, 3, 2, 5, 6])
np.in1d(values, [2, 3, 6])

File input and output with arrays

Storing arrays on disk in binary format



In [ ]:

    
arr = np.arange(10)
np.save('some_array', arr)



In [ ]:

    
np.load('some_array.npy')



In [ ]:

    
np.savez('array_archive.npz', a=arr, b=arr)



In [ ]:

    
arch = np.load('array_archive.npz')
arch['b']



In [ ]:

    
!rm some_array.npy
!rm array_archive.npz

Saving and loading text files



In [ ]:

    
!cat array_ex.txt



In [ ]:

    
arr = np.loadtxt('array_ex.txt', delimiter=',')
arr

Linear algebra



In [ ]:

    
x = np.array([[1., 2., 3.], [4., 5., 6.]])
y = np.array([[6., 23.], [-1, 7], [8, 9]])
x
y
x.dot(y)  # equivalently np.dot(x, y)



In [ ]:

    
np.dot(x, np.ones(3))



In [ ]:

    
np.random.seed(12345)



In [ ]:

    
from numpy.linalg import inv, qr
X = randn(5, 5)
mat = X.T.dot(X)
inv(mat)
mat.dot(inv(mat))
q, r = qr(mat)
r

Random number generation



In [ ]:

    
samples = np.random.normal(size=(4, 4))
samples



In [ ]:

    
from random import normalvariate
N = 1000000
%timeit samples = [normalvariate(0, 1) for _ in xrange(N)]
%timeit np.random.normal(size=N)

Example: Random Walks

import random position = 0 walk = [position] steps = 1000 for i in xrange(steps): step = 1 if random.randint(0, 1) else -1 position += step walk.append(position)



In [ ]:

    
np.random.seed(12345)



In [ ]:

    
nsteps = 1000
draws = np.random.randint(0, 2, size=nsteps)
steps = np.where(draws > 0, 1, -1)
walk = steps.cumsum()



In [ ]:

    
walk.min()
walk.max()



In [ ]:

    
(np.abs(walk) >= 10).argmax()

Simulating many random walks at once



In [ ]:

    
nwalks = 5000
nsteps = 1000
draws = np.random.randint(0, 2, size=(nwalks, nsteps)) # 0 or 1
steps = np.where(draws > 0, 1, -1)
walks = steps.cumsum(1)
walks



In [ ]:

    
walks.max()
walks.min()



In [ ]:

    
hits30 = (np.abs(walks) >= 30).any(1)
hits30
hits30.sum() # Number that hit 30 or -30



In [ ]:

    
crossing_times = (np.abs(walks[hits30]) >= 30).argmax(1)
crossing_times.mean()



In [ ]:

    
steps = np.random.normal(loc=0, scale=0.25,
                         size=(nwalks, nsteps))



In [ ]: