In [1]:
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
Python module for manipulating tabular data
pandasDataFramenumpyResources
pandas?80% of the effort in data analysis is spent cleaning data. Hadley Wickham
Efficency
Raw data is often in the wrong format
scikit-learn interfaceStorage may be best in a different format
Data from:
Baker L, Flemming JEM, Jonsen ID, Lidgard DC, Iverson SJ, Bowen WD (2015) A novel approach to quantifying the spatiotemporal behavior of instrumented grey seals used to sample the environment. Movement Ecology 3(1):20. doi:10.1186/s40462-015-0047-4
Lidgard DC, Bowen WD, Iverson SJ (2015) Data from: A novel approach to quantifying the spatiotemporal behavior of instrumented grey seals used to sample the environment. Movebank Data Repository. doi:10.5441/001/1.910p0c20
In [2]:
f = open("Grey seals (Halichoerus grypus) at Sable Island (data from Baker et al. 2015).csv", 'r')
lines = f.readlines()
lines[:10]
Out[2]:
In [3]:
?pd.read_csv
In [4]:
df = pd.read_csv("Grey seals (Halichoerus grypus) at Sable Island (data from Baker et al. 2015).csv",
parse_dates=[2])
df.head(3)
Out[4]:
In [5]:
df.dtypes
Out[5]:
In [6]:
df.describe()
Out[6]:
In [7]:
len(df)
Out[7]:
In [8]:
df[:10:2]
Out[8]:
In [9]:
df[-5:]
Out[9]:
In [10]:
longitude = df['location-long'].values
print(type(longitude))
print len(longitude)
longitude
Out[10]:
In [11]:
df["individual-local-identifier"].unique()
Out[11]:
In [12]:
df.head(2)
Out[12]:
In [13]:
sdf = df[["timestamp","location-long","location-lat","individual-local-identifier","event-id"]]
sdf.head(5)
Out[13]:
In [14]:
sdf.set_index("timestamp",inplace=True)
sdf.head(5)
Out[14]:
In [15]:
behav = np.random.randn(len(sdf))
sdf.insert(4,'behavior',behav)
sdf.head(5)
Out[15]:
In [16]:
sdf = sdf.rename(columns={"location-long":"longitude",
"location-lat":"latitude",
"individual-local-identifier": "individual"})
sdf.head(5)
Out[16]:
In [17]:
sdf.to_csv("seal-behav.csv")
!head "seal-behav.csv"
In [18]:
sd = sdf.pivot(columns='individual') #row, column, values (optional)
sd[:5]
Out[18]:
In [19]:
sd['behavior'][:5]
Out[19]:
In [20]:
longLat = sdf[['individual', 'longitude', 'latitude']]
longLat[2::5000]
Out[20]:
In [21]:
sd[['longitude', 'latitude']][::5000]
Out[21]:
In [22]:
df[df["individual-local-identifier"] == "F719"][:5]
Out[22]:
In [23]:
sd[[('behavior', "F719"), ('latitude', "F104"),('longitude', "F719")]][:5]
Out[23]:
In [24]:
sd[[('behavior', "F719"),
('latitude', "F719"),
('longitude', "F719")]].dropna()[:5]
Out[24]:
In [25]:
sd.plot()
Out[25]:
In [26]:
sd['behavior'].plot(figsize=(10, 10))
plt.ylabel('behavior')
plt.title('Seal plotting exercise')
plt.savefig('seal_behavior.png')
plt.grid()
plt.show()
In [27]:
sd[('behavior',"K 11")].plot(figsize=(10, 10))
plt.ylabel('behavior')
plt.title('Seal plotting exercise')
plt.savefig('seal-k11.png')
plt.grid()
plt.show()
In [ ]:
In [28]:
import urllib2
import StringIO
import itertools
import datetime
In [29]:
import string
boulder_url = "http://www.esrl.noaa.gov/psd/boulder/data/boulderdaily.complete"
def cdate(x1, x2, x3):
try:
return datetime.datetime(int(x1), int(x2), int(x3))
except:
return datetime.datetime(int(x1), int(x2), int(x3)-1)
df = pd.read_csv(boulder_url, sep=' +', index_col=0, skiprows=1, skipfooter=14,
na_values='-998', engine='python', \
parse_dates=[[0,1,2]], date_parser=cdate, \
header=None,
names=['year', 'month', 'day', 'tmax', 'tmin', 'precip', 'snow', 'snowcover'],)
df.dtypes
Out[29]:
In [30]:
df.tail(5)
Out[30]:
In [32]:
df['tmax'][-365:].plot()
Out[32]:
In [ ]: