In [ ]:

    

import numpy as np
import pandas as pd


    

In [ ]:

    

# split apply combine
df = pd.DataFrame(data={
    'key1': ['a','b','a','a','b'],
    'key2': ['one','two','two','one','two'],
    'data1': np.random.randn(5),
    'data2': np.random.randn(5)
})

df


    

In [ ]:

    

# s1 = df['data1']
# grouped = s1.groupby(df['key1'])
grouped1 = df['data1'].groupby(df['key1'])
grouped1.mean()


    

In [ ]:

    

# give the right length: 5
t1 = ['h', 'j', 'k', 'k', 'h']
grouped2 = df['data1'].groupby(t1)
grouped2.mean()


    

In [ ]:

    

grouped3 = df['data1'].groupby([df['key1'], df['key2']])
grouped3.mean()


    

In [ ]:

    

grouped3.mean().unstack()


    

In [ ]:

    

grouped4 = df['data1'].groupby([df['key1'], df['key2'], t1])
grouped4.mean()


    

In [ ]:

    

grouped4.mean().unstack()


    

In [ ]:

    

grouped5 = df.groupby(['key1'])
grouped5.mean()


    

In [ ]:

    

grouped5.size()


    

In [ ]:

    

grouped6 = df.groupby(['key1', 'key2'])
grouped6.mean()


    

In [ ]:

    

grouped6.size()


    

In [ ]:

    

for name, group in df.groupby(['key1']):
    print(name)
    print(group)

for (k1, k2), group in df.groupby(['key1', 'key2']):
    print(k1, k2)
    print(group)
    
l1 = list(df.groupby(['key1']))
print(l1)
d1 = dict(l1)
# is-a DataFrame type
print('type:', type(d1['a']))

# episode
l2 = [('aa', 12), ('bb', 13)]
d2 = dict(l2)
print(d2)


    

In [ ]:

    

df2 = pd.DataFrame(
    data=np.random.randn(5,5),
    index=['east', 'west', 'north', 'south', 'center'],
    columns=['a', 'b', 'c', 'd', 'e']
)

df2.loc[2:3, ['b','c']] = np.nan

df2


    

In [ ]:

    

mapping = {
    'a': 'red',
    'b': 'blue',
    'c': 'red',
    'd': 'red',
    'e': 'blue',
    'f': 'yellow'
}
by_columns = df2.groupby(by=mapping, axis=1)
by_columns


    

In [ ]:

    

dt = dict(list(by_columns))
dt['red']


    

In [ ]:

    

print(-0.477009+0.767283-0.773931)
by_columns.sum()


    

In [ ]:

    

print((-0.477009+0.767283-0.773931)/3)
by_columns.mean()


    

In [ ]:

    

s1 = pd.Series(data=mapping)
s1


    

In [ ]:

    

df2.groupby(by=s1, axis=1).count()


    

In [ ]:

    

print(df)
idx_len_by_rows = df2.groupby(by=len)
list(idx_len_by_rows)


    

In [ ]:

    

# must right row length: 5
idx_len_other_by_rows = df2.groupby(by=[len, ['o1', 'o2', 'o2', 'o2', 'o1']], axis=0)
idx_len_other_by_rows.sum()


    

In [ ]:

    

cols = pd.MultiIndex.from_arrays([['JP', 'JP', 'JP', 'US', 'US'], [1,1,5,1,3]], names=['cty', 'tenor'])
print(type(cols))
print(cols)
df3 = pd.DataFrame(
    data=np.random.randn(4,5),
    columns=cols
)

# MultiIndex: 
#   labels:
#         the dim-1 is [0:JP, 1:US]
#         the dim-2 is [0:1, 2:5, 1:3]

df3


    

In [ ]:

    

# episode for multiindex
# m-1
idx1 = pd.Index([('A', 'x1'),('A', 'x2'), ('B', 'y1'), ('B', 'y2'), ('B', 'y3')], names=['c1', 'c2'])
idx1
df4 = pd.DataFrame(
    data=np.random.randint(1,10,(5,4)), index=idx1
)
df4


    

In [ ]:

    

# m-2
c1 = ['A', 'A', 'B', 'B', 'B']
c2 = ['x1', 'x2', 'y1', 'y2', 'y3']

idx2 = pd.MultiIndex.from_arrays([c1, c2], names=['c1', 'c2'])
print(idx2)

df5 = pd.DataFrame(
    data=np.random.randint(1,10,(5,4)), index=idx2
)
df5


    

In [ ]:

    

# m-3
idx3 = pd.MultiIndex.from_product([['A', 'B'], ['x1', 'y1']], names=['c1', 'c2'])
print(idx3)
df6 = pd.DataFrame(
    data=np.random.randint(1, 10, (2,4)), columns=idx3
)
df6


    

In [ ]:

    

df


    

In [ ]:

    

grouped = df.groupby('key1')
# print(list(grouped['data1']))
grouped['data1'].quantile(0.9)


    

In [ ]:

    

def peak_to_peak(arr):
    # type is Series
    # print(type(arr))
    return arr.max() - arr.min()

grouped.agg(peak_to_peak)


    

In [ ]:

    

grouped.describe()


    

In [ ]:

    

df_tips = pd.read_csv('./pydata-book/examples/tips.csv')
df_tips[-5:]


    

In [ ]:

    

df_tips['tip_pct'] = df_tips['tip'] / df_tips['total_bill']
df_tips.loc[:5,['tip', 'total_bill', 'tip_pct']]


    

In [ ]:

    

grouped_by_smoker_time = df_tips.groupby(['smoker', 'time'])
grouped_by_smoker_time.agg('mean')


    

In [ ]:

    

tip_pct_of_group = grouped_by_smoker_time['tip_pct']
tip_pct_of_group.agg([('mycount','count')])


    

In [ ]:

    

tip_size_of_group = grouped_by_smoker_time['tip', 'size']
tip_size_of_group.agg('count')


    

In [ ]:

    

tip_size_of_group.agg([('mycount', 'count'), ('mystd', np.std)])


    

In [ ]:

    

tip_size_of_group.agg({'tip': [np.max, np.min], 'size': ['count', 'sum']})