In [12]:
import os
import codecs
import re
from quora import Topic,Question
import pandas as pd
import numpy as np
train_file = 'c:/Users/lxp/Desktop/nlp/quora/train.csv'
df = pd.read_csv(train_file, index_col='id')
df
Out[12]:
qid1
qid2
question1
question2
is_duplicate
id
0
1
2
What is the step by step guide to invest in sh...
What is the step by step guide to invest in sh...
0
1
3
4
What is the story of Kohinoor (Koh-i-Noor) Dia...
What would happen if the Indian government sto...
0
2
5
6
How can I increase the speed of my internet co...
How can Internet speed be increased by hacking...
0
3
7
8
Why am I mentally very lonely? How can I solve...
Find the remainder when [math]23^{24}[/math] i...
0
4
9
10
Which one dissolve in water quikly sugar, salt...
Which fish would survive in salt water?
0
5
11
12
Astrology: I am a Capricorn Sun Cap moon and c...
I'm a triple Capricorn (Sun, Moon and ascendan...
1
6
13
14
Should I buy tiago?
What keeps childern active and far from phone ...
0
7
15
16
How can I be a good geologist?
What should I do to be a great geologist?
1
8
17
18
When do you use シ instead of し?
When do you use "&" instead of "and"?
0
9
19
20
Motorola (company): Can I hack my Charter Moto...
How do I hack Motorola DCX3400 for free internet?
0
10
21
22
Method to find separation of slits using fresn...
What are some of the things technicians can te...
0
11
23
24
How do I read and find my YouTube comments?
How can I see all my Youtube comments?
1
12
25
26
What can make Physics easy to learn?
How can you make physics easy to learn?
1
13
27
28
What was your first sexual experience like?
What was your first sexual experience?
1
14
29
30
What are the laws to change your status from a...
What are the laws to change your status from a...
0
15
31
32
What would a Trump presidency mean for current...
How will a Trump presidency affect the student...
1
16
33
34
What does manipulation mean?
What does manipulation means?
1
17
35
36
Why do girls want to be friends with the guy t...
How do guys feel after rejecting a girl?
0
18
37
38
Why are so many Quora users posting questions ...
Why do people ask Quora questions which can be...
1
19
39
40
Which is the best digital marketing institutio...
Which is the best digital marketing institute ...
0
20
41
42
Why do rockets look white?
Why are rockets and boosters painted white?
1
21
43
44
What's causing someone to be jealous?
What can I do to avoid being jealous of someone?
0
22
45
46
What are the questions should not ask on Quora?
Which question should I ask on Quora?
0
23
47
48
How much is 30 kV in HP?
Where can I find a conversion chart for CC to ...
0
24
49
50
What does it mean that every time I look at th...
How many times a day do a clock’s hands overlap?
0
25
51
52
What are some tips on making it through the jo...
What are some tips on making it through the jo...
0
26
53
54
What is web application?
What is the web application framework?
0
27
55
56
Does society place too much importance on sports?
How do sports contribute to the society?
0
28
57
58
What is best way to make money online?
What is best way to ask for money online?
0
29
59
60
How should I prepare for CA final law?
How one should know that he/she completely pre...
1
...
...
...
...
...
...
404260
182494
691
Which phone is best under 12000?
What is the best phone to buy below 15k?
0
404261
281150
124172
Who is the overall most popular Game of Throne...
Who is the most popular character in the Game ...
1
404262
537905
466328
How do you troubleshoot a Toshiba laptop?
How do I reset a Toshiba laptop?
0
404263
375195
537906
How does the burning of fossil fuels contribut...
Why does CO2 contribute more to global warming...
0
404264
537907
537908
Is it safe to store an external battery power ...
How do I make a safe and cheap power bank?
0
404265
25994
16064
How can I gain weight on my body?
What should I eat to gain weight?
1
404266
155813
146284
What is the green dot next to the phone icon o...
My boyfriend says he deleted his Facebook Mess...
0
404267
20171
290649
What are the causes of the fall of the Roman E...
What were the most important causes and effect...
1
404268
537909
537910
Why don't we still do great music like in the ...
Should I raise my young child on 80's music?
0
404269
537911
349794
How do you diagnose antisocial personality dis...
What Does It Feel Like to have antisocial pers...
0
404270
537912
35364
What is the difference between who and how?
What is the difference between "&" and "and"?
0
404271
537913
537914
Does Stalin have any grandchildren that are st...
What was Joseph Stalin's 5 year plan? How did ...
0
404272
128018
14005
What are the best new car products or inventio...
What are some mind-blowing vehicles tools that...
1
404273
537915
537916
What happens if you put milk in a coffee maker?
What would happen if I put milk instead of wat...
1
404274
178643
87385
Will the next generation of parenting change o...
What kind of parents will the next generation ...
1
404275
97922
537917
In accounting, why do we debit expenses and cr...
What is a utilities expense in accounting? How...
0
404276
24305
308365
What is copilotsearch.com?
What is ContenVania.com?
0
404277
355668
537918
What does analytics do?
What are analytical people like?
0
404278
537919
169786
How did you prepare for AIIMS/NEET/AIPMT?
How did you prepare for the AIIMS UG entrance ...
0
404279
537920
537921
What is the minimum time required to build a f...
What is a cheaper and quicker way to build an ...
0
404280
537922
537923
What are some outfit ideas to wear to a frat p...
What are some outfit ideas wear to a frat them...
1
404281
99131
81495
Why is Manaphy childish in Pokémon Ranger and ...
Why is Manaphy annoying in Pokemon ranger and ...
1
404282
1931
16773
How does a long distance relationship work?
How are long distance relationships maintained?
1
404283
537924
537925
What do you think of the removal of the MagSaf...
What will the CPU upgrade to the 2016 Apple Ma...
0
404284
537926
537927
What does Jainism say about homosexuality?
What does Jainism say about Gays and Homosexua...
1
404285
433578
379845
How many keywords are there in the Racket prog...
How many keywords are there in PERL Programmin...
0
404286
18840
155606
Do you believe there is life after death?
Is it true that there is life after death?
1
404287
537928
537929
What is one coin?
What's this coin?
0
404288
537930
537931
What is the approx annual cost of living while...
I am having little hairfall problem but I want...
0
404289
537932
537933
What is like to have sex with cousin?
What is it like to have sex with your cousin?
0
404290 rows × 5 columns
In [27]:
test_file = 'c:/Users/lxp/Desktop/nlp/quora/test.csv'
df_test = pd.read_csv(train_file, index_col='id')
df_test
Out[27]:
qid1
qid2
question1
question2
is_duplicate
id
0
1
2
What is the step by step guide to invest in sh...
What is the step by step guide to invest in sh...
0
1
3
4
What is the story of Kohinoor (Koh-i-Noor) Dia...
What would happen if the Indian government sto...
0
2
5
6
How can I increase the speed of my internet co...
How can Internet speed be increased by hacking...
0
3
7
8
Why am I mentally very lonely? How can I solve...
Find the remainder when [math]23^{24}[/math] i...
0
4
9
10
Which one dissolve in water quikly sugar, salt...
Which fish would survive in salt water?
0
5
11
12
Astrology: I am a Capricorn Sun Cap moon and c...
I'm a triple Capricorn (Sun, Moon and ascendan...
1
6
13
14
Should I buy tiago?
What keeps childern active and far from phone ...
0
7
15
16
How can I be a good geologist?
What should I do to be a great geologist?
1
8
17
18
When do you use シ instead of し?
When do you use "&" instead of "and"?
0
9
19
20
Motorola (company): Can I hack my Charter Moto...
How do I hack Motorola DCX3400 for free internet?
0
10
21
22
Method to find separation of slits using fresn...
What are some of the things technicians can te...
0
11
23
24
How do I read and find my YouTube comments?
How can I see all my Youtube comments?
1
12
25
26
What can make Physics easy to learn?
How can you make physics easy to learn?
1
13
27
28
What was your first sexual experience like?
What was your first sexual experience?
1
14
29
30
What are the laws to change your status from a...
What are the laws to change your status from a...
0
15
31
32
What would a Trump presidency mean for current...
How will a Trump presidency affect the student...
1
16
33
34
What does manipulation mean?
What does manipulation means?
1
17
35
36
Why do girls want to be friends with the guy t...
How do guys feel after rejecting a girl?
0
18
37
38
Why are so many Quora users posting questions ...
Why do people ask Quora questions which can be...
1
19
39
40
Which is the best digital marketing institutio...
Which is the best digital marketing institute ...
0
20
41
42
Why do rockets look white?
Why are rockets and boosters painted white?
1
21
43
44
What's causing someone to be jealous?
What can I do to avoid being jealous of someone?
0
22
45
46
What are the questions should not ask on Quora?
Which question should I ask on Quora?
0
23
47
48
How much is 30 kV in HP?
Where can I find a conversion chart for CC to ...
0
24
49
50
What does it mean that every time I look at th...
How many times a day do a clock’s hands overlap?
0
25
51
52
What are some tips on making it through the jo...
What are some tips on making it through the jo...
0
26
53
54
What is web application?
What is the web application framework?
0
27
55
56
Does society place too much importance on sports?
How do sports contribute to the society?
0
28
57
58
What is best way to make money online?
What is best way to ask for money online?
0
29
59
60
How should I prepare for CA final law?
How one should know that he/she completely pre...
1
...
...
...
...
...
...
404260
182494
691
Which phone is best under 12000?
What is the best phone to buy below 15k?
0
404261
281150
124172
Who is the overall most popular Game of Throne...
Who is the most popular character in the Game ...
1
404262
537905
466328
How do you troubleshoot a Toshiba laptop?
How do I reset a Toshiba laptop?
0
404263
375195
537906
How does the burning of fossil fuels contribut...
Why does CO2 contribute more to global warming...
0
404264
537907
537908
Is it safe to store an external battery power ...
How do I make a safe and cheap power bank?
0
404265
25994
16064
How can I gain weight on my body?
What should I eat to gain weight?
1
404266
155813
146284
What is the green dot next to the phone icon o...
My boyfriend says he deleted his Facebook Mess...
0
404267
20171
290649
What are the causes of the fall of the Roman E...
What were the most important causes and effect...
1
404268
537909
537910
Why don't we still do great music like in the ...
Should I raise my young child on 80's music?
0
404269
537911
349794
How do you diagnose antisocial personality dis...
What Does It Feel Like to have antisocial pers...
0
404270
537912
35364
What is the difference between who and how?
What is the difference between "&" and "and"?
0
404271
537913
537914
Does Stalin have any grandchildren that are st...
What was Joseph Stalin's 5 year plan? How did ...
0
404272
128018
14005
What are the best new car products or inventio...
What are some mind-blowing vehicles tools that...
1
404273
537915
537916
What happens if you put milk in a coffee maker?
What would happen if I put milk instead of wat...
1
404274
178643
87385
Will the next generation of parenting change o...
What kind of parents will the next generation ...
1
404275
97922
537917
In accounting, why do we debit expenses and cr...
What is a utilities expense in accounting? How...
0
404276
24305
308365
What is copilotsearch.com?
What is ContenVania.com?
0
404277
355668
537918
What does analytics do?
What are analytical people like?
0
404278
537919
169786
How did you prepare for AIIMS/NEET/AIPMT?
How did you prepare for the AIIMS UG entrance ...
0
404279
537920
537921
What is the minimum time required to build a f...
What is a cheaper and quicker way to build an ...
0
404280
537922
537923
What are some outfit ideas to wear to a frat p...
What are some outfit ideas wear to a frat them...
1
404281
99131
81495
Why is Manaphy childish in Pokémon Ranger and ...
Why is Manaphy annoying in Pokemon ranger and ...
1
404282
1931
16773
How does a long distance relationship work?
How are long distance relationships maintained?
1
404283
537924
537925
What do you think of the removal of the MagSaf...
What will the CPU upgrade to the 2016 Apple Ma...
0
404284
537926
537927
What does Jainism say about homosexuality?
What does Jainism say about Gays and Homosexua...
1
404285
433578
379845
How many keywords are there in the Racket prog...
How many keywords are there in PERL Programmin...
0
404286
18840
155606
Do you believe there is life after death?
Is it true that there is life after death?
1
404287
537928
537929
What is one coin?
What's this coin?
0
404288
537930
537931
What is the approx annual cost of living while...
I am having little hairfall problem but I want...
0
404289
537932
537933
What is like to have sex with cousin?
What is it like to have sex with your cousin?
0
404290 rows × 5 columns
In [13]:
questions = dict()
for row in df.iterrows():
questions[row[1]['qid1']] = row[1]['question1']
questions[row[1]['qid2']] = row[1]['question2']
In [23]:
def to_name(q):
q = re.sub(' ','-',q)
q = re.sub('\?','',q)
return q
In [24]:
to_name('What is the step by step guide to invest in share market in india?')
Out[24]:
'What-is-the-step-by-step-guide-to-invest-in-share-market-in-india'
In [25]:
questions.values()[:10]
Out[25]:
['What is the step by step guide to invest in share market in india?',
'What is the step by step guide to invest in share market?',
'What is the story of Kohinoor (Koh-i-Noor) Diamond?',
'What would happen if the Indian government stole the Kohinoor (Koh-i-Noor) diamond back?',
'How can I increase the speed of my internet connection while using a VPN?',
'How can Internet speed be increased by hacking through DNS?',
'Why am I mentally very lonely? How can I solve it?',
'Find the remainder when [math]23^{24}[/math] is divided by 24,23?',
'Which one dissolve in water quikly sugar, salt, methane and carbon di oxide?',
'Which fish would survive in salt water?']
In [26]:
questions_name = [to_name(q) for q in questions.values()]
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-26-8113d29cbff0> in <module>()
----> 1 questions_name = [to_name(q) for q in questions.values()]
<ipython-input-23-e1d48f47e164> in to_name(q)
1 def to_name(q):
----> 2 q = re.sub(' ','-',q)
3 q = re.sub('\?','',q)
4 return q
E:\Anaconda2\lib\re.pyc in sub(pattern, repl, string, count, flags)
153 a callable, it's passed the match object and must return
154 a replacement string to be used."""
--> 155 return _compile(pattern, flags).sub(repl, string, count)
156
157 def subn(pattern, repl, string, count=0, flags=0):
TypeError: expected string or buffer
In [ ]:
Content source: phiedulxp/quora-mining
Similar notebooks: