Analysing tabular data

We are going to use a LIBRARY called numpy


In [1]:
import numpy

In [6]:
numpy.loadtxt(fname='data/data/weather-01.csv', delimiter = ',')


Out[6]:
array([[ 0.,  0.,  1., ...,  3.,  0.,  0.],
       [ 0.,  1.,  2., ...,  1.,  0.,  1.],
       [ 0.,  1.,  1., ...,  2.,  1.,  1.],
       ..., 
       [ 0.,  1.,  1., ...,  1.,  1.,  1.],
       [ 0.,  0.,  0., ...,  0.,  2.,  0.],
       [ 0.,  0.,  1., ...,  1.,  1.,  0.]])

In [3]:
!ls


01-analysing-data.ipynb
Untitled.ipynb
data
data.zip

In [4]:
!ls data


__MACOSX
data

In [5]:
!ls data/data


small-01.csv
small-02.csv
small-03.csv
weather-01.csv
weather-02.csv
weather-03.csv
weather-04.csv
weather-05.csv
weather-06.csv
weather-07.csv
weather-08.csv
weather-09.csv
weather-10.csv
weather-11.csv
weather-12.csv

Variables


In [7]:
weight_kg = 55

In [8]:
print (weight_kg)


55

In [9]:
print('Weight in pounds:', weight_kg*2.2)


Weight in pounds: 121.00000000000001

In [10]:
weight_kg = 57.5

In [11]:
print ('New weight:', weight_kg * 2.2)


New weight: 126.50000000000001

In [12]:
%whos


Variable    Type      Data/Info
-------------------------------
numpy       module    <module 'numpy' from 'C:\<...>ges\\numpy\\__init__.py'>
weight_kg   float     57.5

In [13]:
data = numpy.loadtxt(fname='data/data/weather-01.csv', delimiter = ',')

In [14]:
print (data)


[[ 0.  0.  1. ...,  3.  0.  0.]
 [ 0.  1.  2. ...,  1.  0.  1.]
 [ 0.  1.  1. ...,  2.  1.  1.]
 ..., 
 [ 0.  1.  1. ...,  1.  1.  1.]
 [ 0.  0.  0. ...,  0.  2.  0.]
 [ 0.  0.  1. ...,  1.  1.  0.]]

In [15]:
print (type(data))


<class 'numpy.ndarray'>

In [16]:
%whos


Variable    Type       Data/Info
--------------------------------
data        ndarray    60x40: 2400 elems, type `float64`, 19200 bytes
numpy       module     <module 'numpy' from 'C:\<...>ges\\numpy\\__init__.py'>
weight_kg   float      57.5

In [17]:
# Finding out the data type
print(data.dtype)


float64

In [18]:
# Find out the shape
print(data.shape)


(60, 40)

In [19]:
# This is 60 rows * 40 columns

In [21]:
# Getting a single number out of the array
print ("First value in data:",data[0,0])


First value in data: 0.0

In [23]:
print ('A middle value:', data[30,20])


A middle value: 13.0

In [24]:
# Lets get the first 10 columns for the first 4 rows
print(data[0:4, 0:10])
# Start at index 0 and go up to BUT NOT INCLUDING index 4


[[ 0.  0.  1.  3.  1.  2.  4.  7.  8.  3.]
 [ 0.  1.  2.  1.  2.  1.  3.  2.  2.  6.]
 [ 0.  1.  1.  3.  3.  2.  6.  2.  5.  9.]
 [ 0.  0.  2.  0.  4.  2.  2.  1.  6.  7.]]

In [25]:
# We don't need to start slicing at 0
print (data[5:10, 7:15])


[[  1.   6.   4.   7.   6.   6.   9.   9.]
 [  5.   5.   8.   6.   5.  11.   9.   4.]
 [  3.   5.   3.   7.   8.   8.   5.  10.]
 [  5.   5.   8.   2.   4.  11.  12.  10.]
 [  3.   5.   8.   6.   8.  12.   5.  13.]]

In [26]:
# We don't even need to include the UPPER AND LOWER bounds
smallchunk= data[:3, 36:]
print (smallchunk)


[[ 2.  3.  0.  0.]
 [ 1.  1.  0.  1.]
 [ 2.  2.  1.  1.]]

In [27]:
# Arithmetic on array 
doublesmallchunk = small chunk * 2.0


  File "<ipython-input-27-32655b684dbb>", line 2
    doublesmallchunk = small chunk * 2.0
                                 ^
SyntaxError: invalid syntax

In [28]:
doublesmallchunk = smallchunk * 2.0

In [29]:
print doublesmallchunk


  File "<ipython-input-29-0e0009a1b6ea>", line 1
    print doublesmallchunk
                         ^
SyntaxError: Missing parentheses in call to 'print'

In [30]:
print(doublesmallchunk)


[[ 4.  6.  0.  0.]
 [ 2.  2.  0.  2.]
 [ 4.  4.  2.  2.]]

In [31]:
triplesmallchunk = smallchunk + doublesmallchunk

In [32]:
print (triplesmallchunk)


[[ 6.  9.  0.  0.]
 [ 3.  3.  0.  3.]
 [ 6.  6.  3.  3.]]

In [33]:
print (numpy.mean(data))


6.14875

In [34]:
print (numpy.max(data))


20.0

In [35]:
print (numpy.min(data))


0.0

In [36]:
# Get a set of data for the first station
station_0 = data [0, :]

In [37]:
print (numpy.max(station_0))


18.0

In [38]:
# We don't neet to create 'temporary' array slices
# We can refert to what we call array axes

In [39]:
print (numpy.mean(data, axis = 0)


  File "<ipython-input-39-bfb4915e7437>", line 1
    print (numpy.mean(data, axis = 0)
                                     ^
SyntaxError: unexpected EOF while parsing

In [40]:
# axis = 0 gets the mean DOWN each column, so the mean temperatures for each recording time period
print (numpy.mean(data, axis = 0))


[  0.           0.45         1.11666667   1.75         2.43333333   3.15
   3.8          3.88333333   5.23333333   5.51666667   5.95         5.9
   8.35         7.73333333   8.36666667   9.5          9.58333333
  10.63333333  11.56666667  12.35        13.25        11.96666667
  11.03333333  10.16666667  10.           8.66666667   9.15         7.25
   7.33333333   6.58333333   6.06666667   5.95         5.11666667   3.6
   3.3          3.56666667   2.48333333   1.5          1.13333333
   0.56666667]

In [41]:
# axis = 1 gets the mean ACROSS each row, so the mean temperature for each station across all time periods
print (numpy.mean(data, axis =1))


[ 5.45   5.425  6.1    5.9    5.55   6.225  5.975  6.65   6.625  6.525
  6.775  5.8    6.225  5.75   5.225  6.3    6.55   5.7    5.85   6.55
  5.775  5.825  6.175  6.1    5.8    6.425  6.05   6.025  6.175  6.55
  6.175  6.35   6.725  6.125  7.075  5.725  5.925  6.15   6.075  5.75
  5.975  5.725  6.3    5.9    6.75   5.925  7.225  6.15   5.95   6.275  5.7
  6.1    6.825  5.975  6.725  5.7    6.25   6.4    7.05   5.9  ]

In [42]:
# Do some simple visualisations

In [43]:
import matplotlib.pyplot

In [44]:
%matplotlib inline

In [45]:
image = matplotlib.pyplot.imshow(data)



In [46]:
# let's take a look at the average temperature over time
avg_temperature = numpy.mean(data, axis 0)


  File "<ipython-input-46-c04d750f3bba>", line 2
    avg_temperature = numpy.mean(data, axis 0)
                                            ^
SyntaxError: invalid syntax

In [47]:
avg_temperature = numpy.mean(data, axis = 0)

In [48]:
avg_plot= matplotlib.pyplot.plot(avg_temperature)


Task: produce max and min plots of this data - conclusions


In [49]:
max_temperature = numpy.max(data, axis =0)

In [50]:
print max_temperature


  File "<ipython-input-50-6fb9a0a3183e>", line 1
    print max_temperature
                        ^
SyntaxError: Missing parentheses in call to 'print'

In [51]:
print (max_temperature)


[  0.   1.   2.   3.   4.   5.   6.   7.   8.   9.  10.  11.  12.  13.  14.
  15.  16.  17.  18.  19.  20.  19.  18.  17.  16.  15.  14.  13.  12.  11.
  10.   9.   8.   7.   6.   5.   4.   3.   2.   1.]

In [52]:
min_temperature = numpy.min(data, axis = 0)

In [53]:
max_temperature plot= matplotlib.pyplot.plot(max_temperature)


  File "<ipython-input-53-b827fc544201>", line 1
    max_temperature plot= matplotlib.pyplot.plot(max_temperature)
                       ^
SyntaxError: invalid syntax

In [54]:
max_plot = matplotlib.pyplot.plot(max_temperature)



In [55]:
min_plot = matplotlib.pyplot.plot(min_temperature)



In [56]:
max_plot = matplotlib.pyplot.plot(max_temperature)
min_plot = matplotlib.pyplot.plot(min_temperature)
avg_plot= matplotlib.pyplot.plot(avg_temperature)



In [ ]: