In [22]:
import pandas as pd

train = pd.read_csv("../data/train.csv")
attributes = pd.read_csv("../data/attributes.csv")
descriptions = pd.read_csv("../data/product_descriptions.csv")
all = train.merge(attributes, on="product_uid").merge(descriptions, on="product_uid")
all_grouped = all.groupby("product_uid")

In [ ]:


In [23]:
all['relevance'] = (all['relevance'] - all['relevance'].mean()) / (all['relevance'].max() - all['relevance'].min())

In [17]:
all


Out[17]:
id product_uid product_title search_term relevance name value product_description
0 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet01 Versatile connector for various 90° connection... Not only do angles make joints stronger, they ...
1 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet02 Stronger than angled nailing or screw fastenin... Not only do angles make joints stronger, they ...
2 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet03 Help ensure joints are consistently straight a... Not only do angles make joints stronger, they ...
3 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet04 Dimensions: 3 in. x 3 in. x 1-1/2 in. Not only do angles make joints stronger, they ...
4 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet05 Made from 12-Gauge steel Not only do angles make joints stronger, they ...
5 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet06 Galvanized for extra corrosion resistance Not only do angles make joints stronger, they ...
6 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Bullet07 Install with 10d common nails or #9 x 1-1/2 in... Not only do angles make joints stronger, they ...
7 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Gauge 12 Not only do angles make joints stronger, they ...
8 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Material Galvanized Steel Not only do angles make joints stronger, they ...
9 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 MFG Brand Name Simpson Strong-Tie Not only do angles make joints stronger, they ...
10 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Number of Pieces 1 Not only do angles make joints stronger, they ...
11 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Product Depth (in.) 1.5 Not only do angles make joints stronger, they ...
12 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Product Height (in.) 3 Not only do angles make joints stronger, they ...
13 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Product Weight (lb.) 0.26 Not only do angles make joints stronger, they ...
14 2 100001 Simpson Strong-Tie 12-Gauge Angle angle bracket 0.319279 Product Width (in.) 3 Not only do angles make joints stronger, they ...
15 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet01 Versatile connector for various 90° connection... Not only do angles make joints stronger, they ...
16 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet02 Stronger than angled nailing or screw fastenin... Not only do angles make joints stronger, they ...
17 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet03 Help ensure joints are consistently straight a... Not only do angles make joints stronger, they ...
18 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet04 Dimensions: 3 in. x 3 in. x 1-1/2 in. Not only do angles make joints stronger, they ...
19 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet05 Made from 12-Gauge steel Not only do angles make joints stronger, they ...
20 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet06 Galvanized for extra corrosion resistance Not only do angles make joints stronger, they ...
21 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Bullet07 Install with 10d common nails or #9 x 1-1/2 in... Not only do angles make joints stronger, they ...
22 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Gauge 12 Not only do angles make joints stronger, they ...
23 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Material Galvanized Steel Not only do angles make joints stronger, they ...
24 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 MFG Brand Name Simpson Strong-Tie Not only do angles make joints stronger, they ...
25 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Number of Pieces 1 Not only do angles make joints stronger, they ...
26 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Product Depth (in.) 1.5 Not only do angles make joints stronger, they ...
27 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Product Height (in.) 3 Not only do angles make joints stronger, they ...
28 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Product Weight (lb.) 0.26 Not only do angles make joints stronger, they ...
29 3 100001 Simpson Strong-Tie 12-Gauge Angle l bracket 0.069279 Product Width (in.) 3 Not only do angles make joints stronger, they ...
... ... ... ... ... ... ... ... ...
1410602 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Bullet01 Sheer panel, gently filters light No. 918 Millennial Ryan heathered texture semi...
1410603 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Bullet02 No header pole top panel No. 918 Millennial Ryan heathered texture semi...
1410604 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Bullet03 Machine washable No. 918 Millennial Ryan heathered texture semi...
1410605 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Bullet04 Wide array of colors to choose from No. 918 Millennial Ryan heathered texture semi...
1410606 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Bullet05 100% polyester No. 918 Millennial Ryan heathered texture semi...
1410607 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Bullet06 Contains 1-curtain panel No. 918 Millennial Ryan heathered texture semi...
1410608 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Color Family Green No. 918 Millennial Ryan heathered texture semi...
1410609 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Color/Finish Pool Blue No. 918 Millennial Ryan heathered texture semi...
1410610 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Deco Style Modern No. 918 Millennial Ryan heathered texture semi...
1410611 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Decor Product Type Window Treatment No. 918 Millennial Ryan heathered texture semi...
1410612 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Fabric surface 100% Polyester No. 918 Millennial Ryan heathered texture semi...
1410613 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Features Machine washable No. 918 Millennial Ryan heathered texture semi...
1410614 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Fire retardant No No. 918 Millennial Ryan heathered texture semi...
1410615 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Hanging Style Rod Pocket No. 918 Millennial Ryan heathered texture semi...
1410616 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Included No Additional Items Included No. 918 Millennial Ryan heathered texture semi...
1410617 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Indoor/Outdoor Indoor No. 918 Millennial Ryan heathered texture semi...
1410618 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Insulated No No. 918 Millennial Ryan heathered texture semi...
1410619 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Light Filtering Level Sheer No. 918 Millennial Ryan heathered texture semi...
1410620 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Machine Washable Yes No. 918 Millennial Ryan heathered texture semi...
1410621 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Material Polyester No. 918 Millennial Ryan heathered texture semi...
1410622 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 MFG Brand Name LICHTENBERG No. 918 Millennial Ryan heathered texture semi...
1410623 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Pattern/theme Heathered Texture Sheer No. 918 Millennial Ryan heathered texture semi...
1410624 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Product Depth (in.) 0.1 No. 918 Millennial Ryan heathered texture semi...
1410625 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Product Height (in.) .1 No. 918 Millennial Ryan heathered texture semi...
1410626 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Product Length (in.) 63 No. 918 Millennial Ryan heathered texture semi...
1410627 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Product Weight (lb.) 0.66 No. 918 Millennial Ryan heathered texture semi...
1410628 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Product Width (in.) 40 No. 918 Millennial Ryan heathered texture semi...
1410629 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Width (in) x Length (in) 40 x 63 No. 918 Millennial Ryan heathered texture semi...
1410630 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Window Treatment Product Type Drapery Panel No. 918 Millennial Ryan heathered texture semi...
1410631 221473 206650 LICHTENBERG Pool Blue No. 918 Millennial Ryan ... fine sheer curtain 63 inches -0.015721 Wrinkle resistant No No. 918 Millennial Ryan heathered texture semi...

1410632 rows × 8 columns


In [ ]:
from sklearn.ensemble import RandomForestClassifier

In [32]:
from sklearn import linear_model
clf = linear_model.LinearRegression()
from scipy.sparse import csr_matrix
import numpy as np
trainm = np.random.random((100,100))
trainm[trainm < 0.95] = 0
trainm
strainm = csr_matrix(trainm)
strainm
clf.fit (strainm, [1]*100)
clf.coef_

pd.DataFrame(trainm).to_sparse().as_matrix()


The exact solution is  x = 0                              
Out[32]:
array([[ 0.        ,  0.        ,  0.        , ...,  0.        ,
         0.        ,  0.        ],
       [ 0.        ,  0.        ,  0.        , ...,  0.        ,
         0.        ,  0.        ],
       [ 0.        ,  0.        ,  0.        , ...,  0.        ,
         0.        ,  0.        ],
       ..., 
       [ 0.        ,  0.        ,  0.        , ...,  0.        ,
         0.        ,  0.        ],
       [ 0.        ,  0.        ,  0.        , ...,  0.95643888,
         0.        ,  0.        ],
       [ 0.        ,  0.        ,  0.        , ...,  0.        ,
         0.        ,  0.        ]])

In [ ]:
from sklearn import linear_modeldict ={}
>>> clf = linear_model.LinearRegression()
>>> clf.fit ([[0, 0], [1, 1], [2, 2]], [0, 1, 2])

>>> clf.coef_
array([ 0.5,  0.5])

In [ ]:
all_m = all.as_matrix()
from sklearn.feature_extraction import DictVectorizer
vec = DictVectorizer()

In [34]:
a = pd.concat([all['product_title'], all['value'], all['search_term'], all['product_description'], all['name'] ])
d = {}
for x in a:
    for y in str(x).split():
        d[y] = 1 
print a.shape
print len(d.keys())


(7053160,)
265761

In [ ]:


In [ ]: