In [1]:

    

%matplotlib inline
# import nltk
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# import json
# from nltk.tokenize import WhitespaceTokenizer
# from string import punctuation
# from pymorphy2 import MorphAnalyzer


    

In [9]:

    

data = pd.DataFrame.from_csv('avito_train.tsv', sep='\t')
# data = pd.read_csv('avito_train.tsv', sep='\t', nrows=30000)


    

In [11]:

    

data.head(1)


    

    
Out[11]:






  

    

      

      
category
      
subcategory
      
title
      
description
      
attrs
      
price
      
is_proved
      
is_blocked
      
phones_cnt
      
emails_cnt
      
urls_cnt
      
close_hours
    
    

      
itemid
      

      

      

      

      

      

      

      

      

      

      

      

    
  
  

    

      
10000010
      
Транспорт
      
Автомобили с пробегом
      
Toyota Sera, 1991
      
Новая оригинальная линзованая оптика на ксенон...
      
{"Год выпуска":"1991", "Тип кузова":"Купе", "П...
      
150000
      
NaN
      
0
      
0
      
0
      
0
      
0.03
    
  

In [12]:

    

data.shape


    

    
Out[12]:





(3995803, 12)

In [13]:

    

data.is_proved.unique()


    

    
Out[13]:





array([ nan,   0.,   1.])

In [21]:

    

print("Data info")
print("Non checked: " + str(data.is_proved.isnull().sum()))
print("Prohibited: " + str(len(data[data.is_proved == 0.])))
print("Correct: " + str(len(data[data.is_proved == 1.])))

pd.DataFrame([data.is_proved.isnull().sum(), len(data[data.is_proved == 0.]), ])


    

    




Data info
Non checked: 3720807
Prohibited: 191186
Correct: 83810

In [18]:

    

data.describe()


    

    




C:\Program Files\Anaconda3\lib\site-packages\numpy\lib\function_base.py:3834: RuntimeWarning: Invalid value encountered in percentile
  RuntimeWarning)






    
Out[18]:






  

    

      

      
price
      
is_proved
      
is_blocked
      
phones_cnt
      
emails_cnt
      
urls_cnt
      
close_hours
    
  
  

    

      
count
      
3.995803e+06
      
274996.000000
      
3.995803e+06
      
3.995803e+06
      
3.995803e+06
      
3.995803e+06
      
3.995803e+06
    
    

      
mean
      
6.968911e+05
      
0.304768
      
6.882121e-02
      
1.358348e-01
      
8.543464e-03
      
3.066442e-02
      
8.249453e+00
    
    

      
std
      
6.949002e+06
      
0.460310
      
2.531499e-01
      
8.971230e-01
      
9.516349e-02
      
2.091108e-01
      
8.838389e+00
    
    

      
min
      
0.000000e+00
      
0.000000
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
    
    

      
25%
      
7.000000e+02
      
NaN
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
2.100000e-01
    
    

      
50%
      
6.500000e+03
      
NaN
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
3.900000e+00
    
    

      
75%
      
1.200000e+05
      
NaN
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
0.000000e+00
      
1.742000e+01
    
    

      
max
      
1.000000e+09
      
1.000000
      
1.000000e+00
      
4.280000e+02
      
6.000000e+00
      
3.100000e+01
      
2.400000e+01
    
  

In [22]:

    

data.category.unique()


    

    
Out[22]:





array(['Транспорт', 'Услуги', 'Личные вещи', 'Недвижимость',
       'Бытовая электроника', 'Для дома и дачи', 'Хобби и отдых', 'Работа',
       'Для бизнеса', 'Животные'], dtype=object)

In [24]:

    

data.isnull().any()


    

    
Out[24]:





category       False
subcategory    False
title           True
description     True
attrs           True
price          False
is_proved       True
is_blocked     False
phones_cnt     False
emails_cnt     False
urls_cnt       False
close_hours    False
dtype: bool

In [53]:

    

desc = pd.DataFrame(.append(pd.Series(data.is_proved.isnull().sum()))).T
desc.columns = [0, 1, "NaN"]
desc


    

    
Out[53]:






  

    

      

      
0
      
1
      
NaN
    
  
  

    

      
0
      
191186
      
83810
      
3720807
    
  

In [54]:

    

data.is_blocked.value_counts()


    

    
Out[54]:





0    3720807
1     274996
Name: is_blocked, dtype: int64

In [55]:

    

274996 / 3720807


    

    
Out[55]:





0.07390762272808023

In [ ]: