In [22]:
import pandas as pd
train = pd.read_csv("../data/train.csv")
attributes = pd.read_csv("../data/attributes.csv")
descriptions = pd.read_csv("../data/product_descriptions.csv")
all = train.merge(attributes, on="product_uid").merge(descriptions, on="product_uid")
all_grouped = all.groupby("product_uid")
In [ ]:
In [23]:
all['relevance'] = (all['relevance'] - all['relevance'].mean()) / (all['relevance'].max() - all['relevance'].min())
In [17]:
all
Out[17]:
id
product_uid
product_title
search_term
relevance
name
value
product_description
0
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet01
Versatile connector for various 90° connection...
Not only do angles make joints stronger, they ...
1
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet02
Stronger than angled nailing or screw fastenin...
Not only do angles make joints stronger, they ...
2
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet03
Help ensure joints are consistently straight a...
Not only do angles make joints stronger, they ...
3
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet04
Dimensions: 3 in. x 3 in. x 1-1/2 in.
Not only do angles make joints stronger, they ...
4
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet05
Made from 12-Gauge steel
Not only do angles make joints stronger, they ...
5
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet06
Galvanized for extra corrosion resistance
Not only do angles make joints stronger, they ...
6
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Bullet07
Install with 10d common nails or #9 x 1-1/2 in...
Not only do angles make joints stronger, they ...
7
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Gauge
12
Not only do angles make joints stronger, they ...
8
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Material
Galvanized Steel
Not only do angles make joints stronger, they ...
9
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
MFG Brand Name
Simpson Strong-Tie
Not only do angles make joints stronger, they ...
10
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Number of Pieces
1
Not only do angles make joints stronger, they ...
11
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Product Depth (in.)
1.5
Not only do angles make joints stronger, they ...
12
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Product Height (in.)
3
Not only do angles make joints stronger, they ...
13
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Product Weight (lb.)
0.26
Not only do angles make joints stronger, they ...
14
2
100001
Simpson Strong-Tie 12-Gauge Angle
angle bracket
0.319279
Product Width (in.)
3
Not only do angles make joints stronger, they ...
15
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet01
Versatile connector for various 90° connection...
Not only do angles make joints stronger, they ...
16
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet02
Stronger than angled nailing or screw fastenin...
Not only do angles make joints stronger, they ...
17
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet03
Help ensure joints are consistently straight a...
Not only do angles make joints stronger, they ...
18
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet04
Dimensions: 3 in. x 3 in. x 1-1/2 in.
Not only do angles make joints stronger, they ...
19
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet05
Made from 12-Gauge steel
Not only do angles make joints stronger, they ...
20
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet06
Galvanized for extra corrosion resistance
Not only do angles make joints stronger, they ...
21
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Bullet07
Install with 10d common nails or #9 x 1-1/2 in...
Not only do angles make joints stronger, they ...
22
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Gauge
12
Not only do angles make joints stronger, they ...
23
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Material
Galvanized Steel
Not only do angles make joints stronger, they ...
24
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
MFG Brand Name
Simpson Strong-Tie
Not only do angles make joints stronger, they ...
25
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Number of Pieces
1
Not only do angles make joints stronger, they ...
26
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Product Depth (in.)
1.5
Not only do angles make joints stronger, they ...
27
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Product Height (in.)
3
Not only do angles make joints stronger, they ...
28
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Product Weight (lb.)
0.26
Not only do angles make joints stronger, they ...
29
3
100001
Simpson Strong-Tie 12-Gauge Angle
l bracket
0.069279
Product Width (in.)
3
Not only do angles make joints stronger, they ...
...
...
...
...
...
...
...
...
...
1410602
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Bullet01
Sheer panel, gently filters light
No. 918 Millennial Ryan heathered texture semi...
1410603
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Bullet02
No header pole top panel
No. 918 Millennial Ryan heathered texture semi...
1410604
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Bullet03
Machine washable
No. 918 Millennial Ryan heathered texture semi...
1410605
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Bullet04
Wide array of colors to choose from
No. 918 Millennial Ryan heathered texture semi...
1410606
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Bullet05
100% polyester
No. 918 Millennial Ryan heathered texture semi...
1410607
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Bullet06
Contains 1-curtain panel
No. 918 Millennial Ryan heathered texture semi...
1410608
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Color Family
Green
No. 918 Millennial Ryan heathered texture semi...
1410609
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Color/Finish
Pool Blue
No. 918 Millennial Ryan heathered texture semi...
1410610
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Deco Style
Modern
No. 918 Millennial Ryan heathered texture semi...
1410611
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Decor Product Type
Window Treatment
No. 918 Millennial Ryan heathered texture semi...
1410612
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Fabric surface
100% Polyester
No. 918 Millennial Ryan heathered texture semi...
1410613
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Features
Machine washable
No. 918 Millennial Ryan heathered texture semi...
1410614
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Fire retardant
No
No. 918 Millennial Ryan heathered texture semi...
1410615
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Hanging Style
Rod Pocket
No. 918 Millennial Ryan heathered texture semi...
1410616
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Included
No Additional Items Included
No. 918 Millennial Ryan heathered texture semi...
1410617
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Indoor/Outdoor
Indoor
No. 918 Millennial Ryan heathered texture semi...
1410618
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Insulated
No
No. 918 Millennial Ryan heathered texture semi...
1410619
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Light Filtering Level
Sheer
No. 918 Millennial Ryan heathered texture semi...
1410620
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Machine Washable
Yes
No. 918 Millennial Ryan heathered texture semi...
1410621
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Material
Polyester
No. 918 Millennial Ryan heathered texture semi...
1410622
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
MFG Brand Name
LICHTENBERG
No. 918 Millennial Ryan heathered texture semi...
1410623
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Pattern/theme
Heathered Texture Sheer
No. 918 Millennial Ryan heathered texture semi...
1410624
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Product Depth (in.)
0.1
No. 918 Millennial Ryan heathered texture semi...
1410625
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Product Height (in.)
.1
No. 918 Millennial Ryan heathered texture semi...
1410626
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Product Length (in.)
63
No. 918 Millennial Ryan heathered texture semi...
1410627
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Product Weight (lb.)
0.66
No. 918 Millennial Ryan heathered texture semi...
1410628
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Product Width (in.)
40
No. 918 Millennial Ryan heathered texture semi...
1410629
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Width (in) x Length (in)
40 x 63
No. 918 Millennial Ryan heathered texture semi...
1410630
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Window Treatment Product Type
Drapery Panel
No. 918 Millennial Ryan heathered texture semi...
1410631
221473
206650
LICHTENBERG Pool Blue No. 918 Millennial Ryan ...
fine sheer curtain 63 inches
-0.015721
Wrinkle resistant
No
No. 918 Millennial Ryan heathered texture semi...
1410632 rows × 8 columns
In [ ]:
from sklearn.ensemble import RandomForestClassifier
In [32]:
from sklearn import linear_model
clf = linear_model.LinearRegression()
from scipy.sparse import csr_matrix
import numpy as np
trainm = np.random.random((100,100))
trainm[trainm < 0.95] = 0
trainm
strainm = csr_matrix(trainm)
strainm
clf.fit (strainm, [1]*100)
clf.coef_
pd.DataFrame(trainm).to_sparse().as_matrix()
The exact solution is x = 0
Out[32]:
array([[ 0. , 0. , 0. , ..., 0. ,
0. , 0. ],
[ 0. , 0. , 0. , ..., 0. ,
0. , 0. ],
[ 0. , 0. , 0. , ..., 0. ,
0. , 0. ],
...,
[ 0. , 0. , 0. , ..., 0. ,
0. , 0. ],
[ 0. , 0. , 0. , ..., 0.95643888,
0. , 0. ],
[ 0. , 0. , 0. , ..., 0. ,
0. , 0. ]])
In [ ]:
from sklearn import linear_modeldict ={}
>>> clf = linear_model.LinearRegression()
>>> clf.fit ([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
>>> clf.coef_
array([ 0.5, 0.5])
In [ ]:
all_m = all.as_matrix()
from sklearn.feature_extraction import DictVectorizer
vec = DictVectorizer()
In [34]:
a = pd.concat([all['product_title'], all['value'], all['search_term'], all['product_description'], all['name'] ])
d = {}
for x in a:
for y in str(x).split():
d[y] = 1
print a.shape
print len(d.keys())
(7053160,)
265761
In [ ]:
In [ ]:
Content source: schroeder-dewitt/kaggle_homedepot
Similar notebooks: