notebook.community

Edit and run



In [7]:

    
import json
from pprint import pprint
import re
from os import listdir, SEEK_END
import datetime
import random



In [2]:

    
list_re = [
    r"(\/sarcasm)",
    r"(\[[!?]\](?!\())",
    r"(:\^(?! ?[D)(\[\]]))",
    r"(<\/?sarcasm>)",
    r"(&lt;\/?sarcasm&gt)",
    r"(#sarcasm)",
    r"( \/s(?![a-zA-Z0-9]))"
          ]
sarc = re.compile(r"(\.~)|(\. ~)")
strike = re.compile(r"(?P<strike>~~.*~~)")
sarcasm_re = re.compile('|'.join(list_re))

filename_re = re.compile(r"""(RC_20\d{2}-\d{2}(?!\.bz2))""")



In [3]:

    
def commentGenerator():
    for name in listdir():
        if filename_re.match(name) is not None:
            start = datetime.datetime.now()
            print("Processing file %s, starttime: %s"%(name, start))
            pop = []
            for n, line in enumerate(open(name)):
                try:
                    body = json.loads(line)['body']
                    if 10 <= len(body) <= 300:
                        compareText = body.lower()
                        if(sarcasm_re.search(compareText) is not None) or \
                        ((sarc.search(compareText) is not None) and (strike.search(compareText) is None)):
                            yield (True, body)
                            yield (False, random.choice(pop))
                            pop = []
                        else:
                            pop.append(body)
                            if len(pop) == 1000:
                                yield (False, random.choice(pop))
                                pop = []
                except:
                    print("Error on line %s, : %s"%(n, line))
            print("Finished: " + str(datetime.datetime.now()-start))



In [4]:

    
sarcasticCount = 0
seriousCount = 0
sarcastic = open('json/sarcastic0.json', 'a')
serious = open('json/serious0.json', 'a')
sarcastic.write('[\n')
serious.write('[\n')
for n, (sarcasm, comment) in enumerate(commentGenerator()):
    try:
        if sarcasm:
            json.dump({'text':comment}, sarcastic)
            sarcasticCount += 1
            if sarcasticCount % 100000 == 0:
                serious.write('\n]')
                sarcastic.close()
                sarcastic = open('json/sarcastic%d.json'%sarcasticCount, 'a')
            else:
                sarcastic.write(',\n')
        else:
            json.dump({'text':comment}, serious)
            seriousCount += 1
            if seriousCount % 100000 == 0:
                serious.write('\n]')
                serious.close()
                serious = open('json/serious%d.json'%seriousCount, 'a')
            else:
                serious.write(',\n')
    except:
        print("Error on index %s, : (%s, %s)"%(n, sarcasm, comment))
        sarcastic.close()
        serious.close()
        
sarcastic.close()
serious.close()

sarcasticname = sarcastic.name
seriousname = serious.name

sarcastic.close()
serious.close()

sarcastic = open(sarcasticname, 'r+b')
serious = open(seriousname, 'r+b')
sarcastic.seek(-3, SEEK_END)
sarcastic.write(bytes("\r\n]", 'utf8'))
serious.seek(-3, SEEK_END)
serious.write(bytes("\r\n]", 'utf8'))
sarcastic.close()
serious.close()









    



Processing file RC_2007-10, starttime: 2017-04-18 20:43:36.564866
Finished: 0:00:04.383147
Processing file RC_2007-11, starttime: 2017-04-18 20:43:40.948013
Finished: 0:00:10.006578
Processing file RC_2007-12, starttime: 2017-04-18 20:43:50.954591
Finished: 0:00:10.377783
Processing file RC_2008-01, starttime: 2017-04-18 20:44:01.332374
Finished: 0:00:13.393392
Processing file RC_2008-02, starttime: 2017-04-18 20:44:14.725766
Finished: 0:00:12.478454
Processing file RC_2008-03, starttime: 2017-04-18 20:44:27.204220
Finished: 0:00:12.737296
Processing file RC_2008-04, starttime: 2017-04-18 20:44:39.941516
Finished: 0:00:12.931537
Processing file RC_2008-05, starttime: 2017-04-18 20:44:52.873053
Finished: 0:00:14.713887
Processing file RC_2008-06, starttime: 2017-04-18 20:45:07.586940
Finished: 0:00:15.804731
Processing file RC_2008-07, starttime: 2017-04-18 20:45:23.391671
Finished: 0:00:16.186937
Processing file RC_2008-08, starttime: 2017-04-18 20:45:39.579607
Finished: 0:00:16.393982
Processing file RC_2008-09, starttime: 2017-04-18 20:45:55.973589
Finished: 0:00:18.776603
Processing file RC_2008-10, starttime: 2017-04-18 20:46:14.750192
Finished: 0:00:21.761312
Processing file RC_2008-11, starttime: 2017-04-18 20:46:36.511504
Finished: 0:00:21.472603
Processing file RC_2008-12, starttime: 2017-04-18 20:46:57.984107
Finished: 0:00:22.970110
Processing file RC_2009-01, starttime: 2017-04-18 20:47:20.954217
Finished: 0:00:28.472206
Processing file RC_2009-02, starttime: 2017-04-18 20:47:49.426423
Finished: 0:00:25.848095
Processing file RC_2009-03, starttime: 2017-04-18 20:48:15.274518
Finished: 0:00:29.164905
Processing file RC_2009-04, starttime: 2017-04-18 20:48:44.439423
Finished: 0:00:32.905680
Processing file RC_2009-05, starttime: 2017-04-18 20:49:17.345103
Finished: 0:00:35.023694
Processing file RC_2009-06, starttime: 2017-04-18 20:49:52.368797
Finished: 0:00:36.325480
Processing file RC_2009-07, starttime: 2017-04-18 20:50:28.694277
Finished: 0:00:41.294995
Processing file RC_2009-08, starttime: 2017-04-18 20:51:09.989272
Finished: 0:00:48.077606
Processing file RC_2009-09, starttime: 2017-04-18 20:51:58.066878
Finished: 0:01:00.436297
Processing file RC_2009-10, starttime: 2017-04-18 20:52:58.503175
Finished: 0:01:05.637402
Processing file RC_2009-11, starttime: 2017-04-18 20:54:04.141579
Finished: 0:01:03.208219
Processing file RC_2009-12, starttime: 2017-04-18 20:55:07.349798
Finished: 0:01:16.787390
Processing file RC_2010-01, starttime: 2017-04-18 20:56:24.137688
Finished: 0:01:27.869324
Processing file RC_2010-02, starttime: 2017-04-18 20:57:52.007012
Finished: 0:01:19.088504
Processing file RC_2010-03, starttime: 2017-04-18 20:59:11.096515
Finished: 0:01:29.359915
Processing file RC_2010-04, starttime: 2017-04-18 21:00:40.456430
Finished: 0:01:28.837136
Processing file RC_2010-05, starttime: 2017-04-18 21:02:09.293566
Finished: 0:01:30.104935
Processing file RC_2010-06, starttime: 2017-04-18 21:03:39.399501
Finished: 0:01:37.617378
Processing file RC_2010-07, starttime: 2017-04-18 21:05:17.017879
Finished: 0:01:51.637943
Processing file RC_2010-08, starttime: 2017-04-18 21:07:08.655822
Finished: 0:01:57.585395
Processing file RC_2010-09, starttime: 2017-04-18 21:09:06.241217
Finished: 0:02:09.423270
Processing file RC_2010-10, starttime: 2017-04-18 21:11:15.664487
Finished: 0:02:18.704976
Processing file RC_2010-11, starttime: 2017-04-18 21:13:34.369463
Finished: 0:02:36.920806
Processing file RC_2010-12, starttime: 2017-04-18 21:16:11.290269
Finished: 0:02:45.241814
Processing file RC_2011-01, starttime: 2017-04-18 21:18:56.532083
Finished: 0:03:02.236949
Processing file RC_2011-02, starttime: 2017-04-18 21:21:58.770032
Finished: 0:02:55.880432
Processing file RC_2011-03, starttime: 2017-04-18 21:24:54.650464
Finished: 0:03:28.135502
Processing file RC_2011-04, starttime: 2017-04-18 21:28:22.785966
Finished: 0:03:26.796577
Processing file RC_2011-05, starttime: 2017-04-18 21:31:49.582543
Finished: 0:04:00.227860
Processing file RC_2011-06, starttime: 2017-04-18 21:35:49.810403
Finished: 0:04:36.716122
Processing file RC_2011-07, starttime: 2017-04-18 21:40:26.527525
Finished: 0:04:54.821178
Processing file RC_2011-08, starttime: 2017-04-18 21:45:21.348703
Finished: 0:05:37.806757
Processing file RC_2011-09, starttime: 2017-04-18 21:50:59.155460
Finished: 0:05:31.106184
Processing file RC_2011-10, starttime: 2017-04-18 21:56:30.261644
Finished: 0:06:06.432596
Processing file RC_2011-11, starttime: 2017-04-18 22:02:36.694240
Finished: 0:06:35.134787
Processing file RC_2011-12, starttime: 2017-04-18 22:09:11.829027
Finished: 0:07:01.337532
Processing file RC_2012-01, starttime: 2017-04-18 22:16:13.167059
Finished: 0:07:57.676532
Processing file RC_2012-02, starttime: 2017-04-18 22:24:10.844090
Finished: 0:07:59.504054
Processing file RC_2012-03, starttime: 2017-04-18 22:32:10.348644
Finished: 0:09:00.513049
Processing file RC_2012-04, starttime: 2017-04-18 22:41:10.862193
Finished: 0:08:48.312965
Processing file RC_2012-05, starttime: 2017-04-18 22:49:59.175158
Finished: 0:09:21.840149
Processing file RC_2012-06, starttime: 2017-04-18 22:59:21.015307
Finished: 0:10:07.532875
Processing file RC_2012-07, starttime: 2017-04-18 23:09:28.548690
Finished: 0:11:30.189042
Processing file RC_2012-08, starttime: 2017-04-18 23:20:58.737732
Finished: 0:12:09.214776
Processing file RC_2012-09, starttime: 2017-04-18 23:33:07.953504
Finished: 0:10:38.324968
Processing file RC_2012-10, starttime: 2017-04-18 23:43:46.279472
Finished: 0:11:15.279984
Processing file RC_2012-11, starttime: 2017-04-18 23:55:01.559456
Finished: 0:11:10.640256
Processing file RC_2012-12, starttime: 2017-04-19 00:06:12.199712
Finished: 0:11:45.346293
Processing file RC_2013-01, starttime: 2017-04-19 00:17:57.546005
Error on line 12075789, : {"id":"c7w8iby","distinguished":null,"subreddit":"WTF","author_flair_text":null,"downs":0,"edited":false,"author":"wasweissic
Finished: 0:05:26.370734
Processing file RC_2013-02, starttime: 2017-04-19 00:23:23.916739
Finished: 0:12:31.132123
Processing file RC_2013-03, starttime: 2017-04-19 00:35:55.048862
Finished: 0:14:33.143142
Processing file RC_2013-04, starttime: 2017-04-19 00:50:28.192004
Finished: 0:15:43.740612
Processing file RC_2013-05, starttime: 2017-04-19 01:06:11.932616
Finished: 0:15:03.443855
Processing file RC_2013-06, starttime: 2017-04-19 01:21:15.376471
Finished: 0:14:54.870144
Processing file RC_2013-07, starttime: 2017-04-19 01:36:10.247615
Finished: 0:15:55.991407
Processing file RC_2013-08, starttime: 2017-04-19 01:52:06.240022
Finished: 0:16:01.523760
Processing file RC_2013-09, starttime: 2017-04-19 02:08:07.763782
Finished: 0:14:33.451375
Processing file RC_2013-10, starttime: 2017-04-19 02:22:41.215157
Finished: 0:16:17.317724
Processing file RC_2013-11, starttime: 2017-04-19 02:38:58.532881
Finished: 0:16:48.871604
Processing file RC_2013-12, starttime: 2017-04-19 02:55:47.404485
Finished: 0:17:53.115460
Processing file RC_2014-01, starttime: 2017-04-19 03:13:40.519945
Finished: 0:20:02.606139
Processing file RC_2014-02, starttime: 2017-04-19 03:33:43.126084
Finished: 0:18:27.268341
Processing file RC_2014-03, starttime: 2017-04-19 03:52:10.394425
Finished: 0:20:06.032634
Processing file RC_2014-04, starttime: 2017-04-19 04:12:16.427059
Finished: 0:20:07.671223
Processing file RC_2014-05, starttime: 2017-04-19 04:32:24.098282
Finished: 0:20:10.246514
Processing file RC_2014-06, starttime: 2017-04-19 04:52:34.344796
Finished: 0:19:59.911059
Processing file RC_2014-07, starttime: 2017-04-19 05:12:34.255855
Finished: 0:22:15.458700
Processing file RC_2014-08, starttime: 2017-04-19 05:34:49.714555
Finished: 0:22:06.948276
Processing file RC_2014-09, starttime: 2017-04-19 05:56:56.662831
Finished: 0:20:36.119162
Processing file RC_2014-10, starttime: 2017-04-19 06:17:32.782993
Finished: 0:21:48.940773
Processing file RC_2014-11, starttime: 2017-04-19 06:39:21.723766
Finished: 0:21:11.670174
Processing file RC_2014-12, starttime: 2017-04-19 07:00:33.393940
Finished: 0:22:22.229815
Processing file RC_2015-01, starttime: 2017-04-19 07:22:55.623755
Finished: 0:24:49.752807
Processing file RC_2015-02, starttime: 2017-04-19 07:47:45.376562
Finished: 0:22:28.138716
Processing file RC_2015-03, starttime: 2017-04-19 08:10:13.515278
Finished: 0:26:55.485840
Processing file RC_2015-04, starttime: 2017-04-19 08:37:09.001118
Finished: 0:28:50.431441
Processing file RC_2015-05, starttime: 2017-04-19 09:05:59.432559
Finished: 0:27:39.150709



In [5]:

    
sarcastic.close()
serious.close()