In [6]:

    

from lib.modules_hotstar import describe_data, processinput_data, extract_data_fields, process_target, transform_countdata


    

In [7]:

    

train_X, train_Y,  test, details = describe_data('data/hotstar')


    

    




Directory Read from:data/hotstar/*

Files Read:

data/hotstar/train_data.json
data/hotstar/test_data.json
data/hotstar/sample_submission.csv

In [5]:

    

genre_list = extract_data_fields(train_X, 'genres')
city_list = extract_data_fields(train_X, 'cities')
train_Y = process_target(train_Y)
daytime_list = range(1, 25)
weekday_list = range(1, 8)


    

    




Drama:164,Cricket:3889,Comedy:8 list completed
kolkata:26,bangalore:2308,delhi:898,mumbai:828 list completed

In [9]:

    

label = test['ID']
label.to_csv('label.csv', index=False, header='label')


    

In [7]:

    

import pandas as pd
df = pd.DataFrame()
for col in genre_list:
    df[col] = 0
for col in city_list:
    df[col] = 0
for col in daytime_list:
    df[col] = 0
for col in weekday_list:
    df[col] = 0


    

In [8]:

    

train_X['genres'][1]


    

    
Out[8]:





u'Cricket:15640,Wildlife:730'

In [9]:

    

new_pd = processinput_data(train_X, df)
test_pd = processinput_data(test, df)


    

    




(200000, 1417)
[ 0.  0.  0. ...,  0.  0.  0.]
Genres:

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
100000 Row completed
110000 Row completed
120000 Row completed
130000 Row completed
140000 Row completed
150000 Row completed
160000 Row completed
170000 Row completed
180000 Row completed
190000 Row completed
Cities 

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
100000 Row completed
110000 Row completed
120000 Row completed
130000 Row completed
140000 Row completed
150000 Row completed
160000 Row completed
170000 Row completed
180000 Row completed
190000 Row completed
Day of week 

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
100000 Row completed
110000 Row completed
120000 Row completed
130000 Row completed
140000 Row completed
150000 Row completed
160000 Row completed
170000 Row completed
180000 Row completed
190000 Row completed
Time of Day 

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
100000 Row completed
110000 Row completed
120000 Row completed
130000 Row completed
140000 Row completed
150000 Row completed
160000 Row completed
170000 Row completed
180000 Row completed
190000 Row completed
(100000, 1417)
[ 0.  0.  0. ...,  0.  0.  0.]
Genres:

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
Cities 

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
Day of week 

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed
Time of Day 

0 Row completed
10000 Row completed
20000 Row completed
30000 Row completed
40000 Row completed
50000 Row completed
60000 Row completed
70000 Row completed
80000 Row completed
90000 Row completed

In [28]:

    

new_pd = pd.read_csv('/home/suraj/Repositories/IndiaHacks ML Hackathon/train_data.csv')
test_pd = pd.read_csv('/home/suraj/Repositories/IndiaHacks ML Hackathon/test_data.csv')


    

In [29]:

    




    

    





ImportErrorTraceback (most recent call last)
<ipython-input-29-e842ade09629> in <module>()
----> 1 from lib.modules_hotstar import add_timeandday_features
      2 train_pd = add_timeandday_features(train_X, new_pd)
      3 test_pd = add_timeandday_features(test, test_pd)

ImportError: cannot import name add_timeandday_features

In [27]:

    




    

    
Out[27]:





Unnamed: 0                  0.0
Cricket                 82379.0
Kabaddi                   255.0
Reality                  4751.0
Wildlife                    0.0
LiveTV                      0.0
Football                    0.0
TalkShow                    0.0
Drama                       0.0
Action                      0.0
Romance                     0.0
Mythology                   0.0
Comedy                      0.0
Kids                        0.0
Crime                       0.0
Family                      0.0
Sport                       0.0
Teen                        0.0
Awards                      0.0
Thriller                    0.0
Badminton                   0.0
Horror                      0.0
Travel                      0.0
Documentary                 0.0
Science                     0.0
Hockey                      0.0
Formula1                    0.0
Table Tennis                0.0
Tennis                      0.0
Athletics                   0.0
                         ...   
monmouth junction           0.0
tumkur                      0.0
northampton                 0.0
bayrut                      0.0
bari                        0.0
bharuch                     0.0
rodgau                      0.0
frankfurt-fechenheim        0.0
palaiseau                   0.0
harare                      0.0
girona                      0.0
licheng                     0.0
san vendemiano              0.0
poitiers                    0.0
wallingford                 0.0
aalborg                     0.0
wallington                  0.0
belfast                     0.0
eivissa                     0.0
minhang qu                  0.0
streatham                   0.0
zhenruzhen                  0.0
pavlodar                    0.0
louisville                  0.0
title_count                60.0
tod                        14.0
dow                         7.0
city                        2.0
genre                       3.0
day                         0.0
Name: 0, dtype: float64

In [85]:

    

title_train = transform_countdata(train_X, 'titles')
title_test = transform_countdata(test, 'titles')


    

In [86]:

    

new_pd['title_count'] = title_train


    

In [87]:

    

test_pd['title_count'] = title_test


    

In [89]:

    

tod_train = transform_countdata(train_X, 'tod')
tod_test = transform_countdata(test, 'tod')
dow_train = transform_countdata(train_X, 'dow')
dow_test = transform_countdata(test, 'dow')

new_pd['tod'] = tod_train
test_pd['tod'] = tod_test
new_pd['dow'] = dow_train
test_pd['dow'] = dow_test


    

In [91]:

    

new_pd.to_csv('train_X.csv')
test_pd.to_csv('test_X.csv')


    

In [4]:

    

import pandas as pd
new_pd = pd.read_csv('/home/suraj/Repositories/IndiaHacks ML Hackathon/train_X.csv')
test_pd = pd.read_csv('/home/suraj/Repositories/IndiaHacks ML Hackathon/test_X.csv')


    

In [ ]:

    

from lib.modules_hotstar import add_timeandday_features
train_pd = add_timeandday_features(train_X, new_pd)
test_pd = add_timeandday_features(test, test_pd)


    

In [14]:

    

city_train = transform_countdata(train_X, 'cities')
city_test = transform_countdata(test, 'cities')
genre_train = transform_countdata(train_X, 'genres')
genre_test = transform_countdata(test, 'genres')

new_pd['city'] = city_train
test_pd['city'] = city_test
new_pd['genre'] = genre_train
test_pd['genre'] = genre_test


    

In [15]:

    

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
# from sklearn.svm import SVC
import pandas as pd


    

In [ ]:

    




    

In [16]:

    

model = GradientBoostingClassifier(n_estimators=100, verbose=True)
model.fit(new_pd, train_Y)


    

    




      Iter       Train Loss   Remaining Time 
         1           0.5244           10.91m
         2           0.5135           10.73m
         3           0.5049           10.67m
         4           0.4973           10.60m
         5           0.4910           10.42m
         6           0.4857           10.25m
         7           0.4812           10.11m
         8           0.4775           10.26m
         9           0.4742           10.23m
        10           0.4714           10.05m
        20           0.4568            8.14m
        30           0.4510            6.65m
        40           0.4481            5.36m
        50           0.4462            4.29m
        60           0.4447            3.38m
        70           0.4435            2.47m
        80           0.4426            1.60m
        90           0.4418           46.64s
       100           0.4412            0.00s






    
Out[16]:





GradientBoostingClassifier(criterion='friedman_mse', init=None,
              learning_rate=0.1, loss='deviance', max_depth=3,
              max_features=None, max_leaf_nodes=None,
              min_impurity_split=1e-07, min_samples_leaf=1,
              min_samples_split=2, min_weight_fraction_leaf=0.0,
              n_estimators=100, presort='auto', random_state=None,
              subsample=1.0, verbose=True, warm_start=False)

In [17]:

    

results = model.predict_proba(test_pd)


    

In [18]:

    

results


    

    
Out[18]:





array([[ 0.97544942,  0.02455058],
       [ 0.94883692,  0.05116308],
       [ 0.84818481,  0.15181519],
       ..., 
       [ 0.98907433,  0.01092567],
       [ 0.98608116,  0.01391884],
       [ 0.9457619 ,  0.0542381 ]])

In [19]:

    

# # uncomment to do grid search - could get better score
# from sklearn.model_selection import train_test_split
# X_train, X_val, y_train, y_val = train_test_split(train_X, train_Y, train_size=0.7, random_state=1)
# 
# # doing grid search for parameters
# from sklearn.grid_search import GridSearchCV
# from sklearn.metrics import roc_auc_score, make_scorer
# clf_scorer = make_scorer(roc_auc_score)
# rfc = GradientBoostingClassifier(verbose=True)
# param_grid = {
#     'max_depth': [4, 8,  12],
#     'max_features': ['sqrt', 'log2', None],
#     'n_estimators': [250, 500, 1000]
# }
# cv_rfc = GridSearchCV(estimator=rfc, param_grid=param_grid, cv=5, scoring=clf_scorer, n_jobs=-1)
# cv_rfc.fit(X_train, y_train)


    

In [20]:

    

results = pd.DataFrame(columns=['seg', 'segment'], data=results)


    

In [21]:

    

results.drop('seg', axis=1, inplace=True)


    

In [22]:

    

results.head()
results['ID'] = label
results = results[['ID', 'segment']]


    

In [23]:

    

results.head()


    

    
Out[23]:






  

    

      

      
ID
      
segment
    
  
  

    

      
0
      
test-1
      
0.024551
    
    

      
1
      
test-10
      
0.051163
    
    

      
2
      
test-100
      
0.151815
    
    

      
3
      
test-1000
      
0.146270
    
    

      
4
      
test-10000
      
0.137074
    
  

In [24]:

    

results.to_csv('/home/suraj/Repositories/IndiaHacks ML Hackathon/submissions/hotstarsub1.csv', index=False)


    

In [1]:

    

label.to_csv('label.csv')


    

    





NameErrorTraceback (most recent call last)
<ipython-input-1-a2f459fbd379> in <module>()
----> 1 label.to_csv('label.csv')

NameError: name 'label' is not defined

In [ ]: