In [1]:

    

import json
import datetime
# Necessary imports 
import sys
home_directory = '/dfs/scratch2/fcipollone'
sys.path.append(home_directory)
import os
import time
from nbminer.notebook_miner import NotebookMinerString
from nbminer.cells.cells import Cell
from nbminer.features.features import Features
from nbminer.stats.summary import Summary
from nbminer.stats.multiple_summary import MultipleSummary
from nbminer.encoders.ast_graph.ast_graph import *
from nbminer.pipeline.pipeline import Pipeline
from nbminer.features.features import Features
from nbminer.preprocess.get_ast_features import GetASTFeatures
from nbminer.preprocess.get_imports import GetImports
from nbminer.preprocess.get_docstrings import GetDocstrings
from nbminer.preprocess.resample_by_node import ResampleByNode
from nbminer.encoders.ast_graph.postorder_hash import PostorderHash, EquivalenceHash
from nbminer.encoders.ast_graph.function_hash import FunctionHash
from lib2to3 import refactor


notebooks = []
filename = '/dfs/scratch2/fcipollone/stackoverflow/guesslang_and_ast/Posts_Reduced.xml'
output_file = open('/dfs/scratch2/fcipollone/stackoverflow/guesslang_and_ast/Posts_EquivalenceHash.xml','w')

count = 0

gastf = GetASTFeatures()
rbn = ResampleByNode()
gi = GetImports()
ph = EquivalenceHash(None, split_call=True, hash_many=False)
#pipe = Pipeline([gastf, rbn, gi, ph])
pipe = Pipeline([gastf, rbn, gi, ph])
start = time.time()

myfixes = ['lib2to3.fixes.fix_' + el for el in (refactor.get_all_fix_names("lib2to3.fixes"))]
refac = refactor.RefactoringTool(myfixes)
total_posts = 0
total_true_true_posts = 0
total_code_blocks = 0
total_true_true_code_blocks = 0
total_lines_of_code = 0
total_workable_codeblocks = 0
#4110319 total lines
for line in open(filename):
    total_posts += 1
    line_obj = json.loads(line)
    true_true_post = False
    new_code_block = False

    for el in line_obj['CodeBlocks']:
        total_code_blocks += 1
        code = el['code']
        if el['Guesslang'].lower().strip() != 'python' or el['Parsable'] != 'True':
            continue
        if not true_true_post:
            true_true_post = True
            total_true_true_posts += 1
        total_true_true_code_blocks += 1
        #### Determine if this is python two or three
        parsable = True
        try:
            ast.parse(code)
        except:
            try:
                tree = refac.refactor_string(code + '\n', 'cell')
                new_code = str(tree)[:-1]
                ast.parse(new_code)
                code = new_code
            except:
                parsable = False
        hashes = []
        imports = {}
        functions = []
        if parsable:
            total_workable_codeblocks += 1
            #total_through_pipe += 1
            nb = NotebookMinerString([code])
            a = Features([nb])
            a = pipe.transform(a)
            imports = a.get_notebook(0).get_feature('imports')
            functions = []
            for seg in a.get_list_segments():
                functions.extend(seg.get_feature('short_name_string'))
            total_lines_of_code += a.get_notebook(0).get_number_cells()
            for seg in a.get_notebook(0).cell_array:
                hashes.append(seg.get_feature('hash'))
        el['hashes'] = hashes
        el['imports'] = imports
        el['functions'] = functions
        new_code_block = True
    #if not new_code_block:
    #    continue
    
    json.dump(line_obj, output_file)
    output_file.write('\n')        
    count += 1
    if count % 10000 == 0:
        end = time.time()
        print (count)
        #print("Time so far per AST: ",(end-start)/total_through_pipe)
        #print("Time so far per post: ",(end-start)/count)
        print ("Estimated time is,",(end-start)/(count/4110319)/3600)


    

    




10000
Estimated time is, 2.3845552704539825
20000
Estimated time is, 2.3665424600660465
30000
Estimated time is, 2.3533850426417833
40000
Estimated time is, 2.4397805415272167
50000
Estimated time is, 2.4130634159251376
60000
Estimated time is, 2.418895204760984
70000
Estimated time is, 2.4535308949132113
80000
Estimated time is, 2.4973307590087996
90000
Estimated time is, 2.5102194095833754
100000
Estimated time is, 2.52349734747136
110000
Estimated time is, 2.5295184029963735
120000
Estimated time is, 2.5254588882273756
130000
Estimated time is, 2.5453424592839844
140000
Estimated time is, 2.5392653527166895
150000
Estimated time is, 2.556394592257759
160000
Estimated time is, 2.52751105525143
170000
Estimated time is, 2.5517913426355263
180000
Estimated time is, 2.5680853352386492
190000
Estimated time is, 2.5389798723649117
200000
Estimated time is, 2.5258551652830623
210000
Estimated time is, 2.549377135660203
220000
Estimated time is, 2.5335596995253646
230000
Estimated time is, 2.5721894646318604
240000
Estimated time is, 2.5739918198277745
250000
Estimated time is, 2.6011593628705465
260000
Estimated time is, 2.591444132652439
270000
Estimated time is, 2.626044621795223
280000
Estimated time is, 2.6137729547363606
290000
Estimated time is, 2.5982684821190922
300000
Estimated time is, 2.5801398681496606
310000
Estimated time is, 2.593752095994528
320000
Estimated time is, 2.5798497513492795
330000
Estimated time is, 2.569040126755339
340000
Estimated time is, 2.5592116231747832
350000
Estimated time is, 2.575248711620758
360000
Estimated time is, 2.565202543304937
370000
Estimated time is, 2.5521720948924878
380000
Estimated time is, 2.543139211768098
390000
Estimated time is, 2.5712651521026673
400000
Estimated time is, 2.5617819894643765
410000
Estimated time is, 2.5527368861902087
420000
Estimated time is, 2.5474369119837577
430000
Estimated time is, 2.570150070845345
440000
Estimated time is, 2.563807740040422
450000
Estimated time is, 2.5618203484331943
460000
Estimated time is, 2.5567303976278715
470000
Estimated time is, 2.552986338189484
480000
Estimated time is, 2.577771672796041
490000
Estimated time is, 2.5671279682788897
500000
Estimated time is, 2.557305520624011
510000
Estimated time is, 2.5566198337520722
520000
Estimated time is, 2.5489884981827684
530000
Estimated time is, 2.549946431470215
540000
Estimated time is, 2.57681439853724
550000
Estimated time is, 2.571331362098725
560000
Estimated time is, 2.5706784213744336
570000
Estimated time is, 2.564508694492398
580000
Estimated time is, 2.5581572265561374
590000
Estimated time is, 2.588275476512629
600000
Estimated time is, 2.5840123111951856
610000
Estimated time is, 2.5818286534592545
620000
Estimated time is, 2.5766249862713777
630000
Estimated time is, 2.572456604297816
640000
Estimated time is, 2.574331146869775
650000
Estimated time is, 2.567736933939604
660000
Estimated time is, 2.5976959384218783
670000
Estimated time is, 2.5963457813509336
680000
Estimated time is, 2.593687773136045
690000
Estimated time is, 2.5891887296777414
700000
Estimated time is, 2.5819692716598
710000
Estimated time is, 2.579630826928026
720000
Estimated time is, 2.5769406361601805
730000
Estimated time is, 2.6080954491333332
740000
Estimated time is, 2.6083529332803534
750000
Estimated time is, 2.608727477186202
760000
Estimated time is, 2.607775131888534
770000
Estimated time is, 2.604259672031442
780000
Estimated time is, 2.600545555773356
790000
Estimated time is, 2.5985506783739036
800000
Estimated time is, 2.6392841374998617
810000
Estimated time is, 2.6430371715449716
820000
Estimated time is, 2.6442999508211984
830000
Estimated time is, 2.64457418483982
840000
Estimated time is, 2.645805869981346
850000
Estimated time is, 2.6468808878526073
860000
Estimated time is, 2.643683012997789
870000
Estimated time is, 2.644182600286199
880000
Estimated time is, 2.6839392040959957
890000
Estimated time is, 2.6827738959630265
900000
Estimated time is, 2.684486085784628
910000
Estimated time is, 2.6827605486831696
920000
Estimated time is, 2.6823163500385494
930000
Estimated time is, 2.6828763104510265
940000
Estimated time is, 2.683904568423126
950000
Estimated time is, 2.683359139777996
960000
Estimated time is, 2.6817434819724824
970000
Estimated time is, 2.7194817132062923
980000
Estimated time is, 2.719890780460658
990000
Estimated time is, 2.7181079138866653
1000000
Estimated time is, 2.716284234362284
1010000
Estimated time is, 2.716026542014525
1020000
Estimated time is, 2.7146319147506377
1030000
Estimated time is, 2.7158701291455616
1040000
Estimated time is, 2.7191455307517987
1050000
Estimated time is, 2.721062652758076
1060000
Estimated time is, 2.7205902365414127
1070000
Estimated time is, 2.757707427736026
1080000
Estimated time is, 2.7577174016277564
1090000
Estimated time is, 2.7577098511253597
1100000
Estimated time is, 2.75829161680125
1110000
Estimated time is, 2.757111289975023
1120000
Estimated time is, 2.756869743119546
1130000
Estimated time is, 2.754460537510202
1140000
Estimated time is, 2.752389222019223
1150000
Estimated time is, 2.7532558504810076
1160000
Estimated time is, 2.7516916739156567
1170000
Estimated time is, 2.78980255422076
1180000
Estimated time is, 2.7915221827896186
1190000
Estimated time is, 2.791742416309628
1200000
Estimated time is, 2.7923223620118183
1210000
Estimated time is, 2.793538687372223
1220000
Estimated time is, 2.797532711741373
1230000
Estimated time is, 2.799494641791558
1240000
Estimated time is, 2.7999839798390522
1250000
Estimated time is, 2.800796761011963
1260000
Estimated time is, 2.8033506344868204
1270000
Estimated time is, 2.80585685306405
1280000
Estimated time is, 2.8441479287625904
1290000
Estimated time is, 2.8460776302195194
1300000
Estimated time is, 2.8452176444572377
1310000
Estimated time is, 2.84538917574998
1320000
Estimated time is, 2.8451608177670864
1330000
Estimated time is, 2.844544274654858
1340000
Estimated time is, 2.843890755337235
1350000
Estimated time is, 2.8430569166932305
1360000
Estimated time is, 2.84192486279707
1370000
Estimated time is, 2.840082115640102
1380000
Estimated time is, 2.839552307817231
1390000
Estimated time is, 2.8403488335623392
1400000
Estimated time is, 2.8815927577879434
1410000
Estimated time is, 2.8850909609574376
1420000
Estimated time is, 2.8859529083249162
1430000
Estimated time is, 2.8857603736059616
1440000
Estimated time is, 2.8889325861611677
1450000
Estimated time is, 2.887537720923735
1460000
Estimated time is, 2.887906270734958
1470000
Estimated time is, 2.8866487078694005
1480000
Estimated time is, 2.885486032507303
1490000
Estimated time is, 2.885509142877116
1500000
Estimated time is, 2.8873015436914176
1510000
Estimated time is, 2.8874854234109826
1520000
Estimated time is, 2.887586239409838
1530000
Estimated time is, 2.8863685741681784
1540000
Estimated time is, 2.9265535836278698
1550000
Estimated time is, 2.928490822734511
1560000
Estimated time is, 2.9292927442971055
1570000
Estimated time is, 2.929009674862023
1580000
Estimated time is, 2.9292564507363448
1590000
Estimated time is, 2.9295054894404573
1600000
Estimated time is, 2.9295614622148647
1610000
Estimated time is, 2.9307352473222266
1620000
Estimated time is, 2.9305155845480626
1630000
Estimated time is, 2.929189209069609
1640000
Estimated time is, 2.9287004414331954
1650000
Estimated time is, 2.9294587375584853
1660000
Estimated time is, 2.9286186634700146
1670000
Estimated time is, 2.9265271448634076
1680000
Estimated time is, 2.9265764272363586
1690000
Estimated time is, 2.9250300749569704
1700000
Estimated time is, 2.965483612152869
1710000
Estimated time is, 2.9665465665310955
1720000
Estimated time is, 2.9674722984951676
1730000
Estimated time is, 2.966641649853751
1740000
Estimated time is, 2.9658177757398305
1750000
Estimated time is, 2.9645097993405343
1760000
Estimated time is, 2.9663217980361667
1770000
Estimated time is, 2.966113540341475
1780000
Estimated time is, 2.9661815711434465
1790000
Estimated time is, 2.965159489456631
1800000
Estimated time is, 2.96497206253349
1810000
Estimated time is, 2.9632951116852464
1820000
Estimated time is, 2.9633305178880933
1830000
Estimated time is, 2.96314027063755
1840000
Estimated time is, 2.9641373184714217
1850000
Estimated time is, 2.963872451078522
1860000
Estimated time is, 2.965856981700257
1870000
Estimated time is, 3.0078655554026623
1880000
Estimated time is, 3.0086765224393504
1890000
Estimated time is, 3.009278223076104
1900000
Estimated time is, 3.009064660753004
1910000
Estimated time is, 3.0092355883385835
1920000
Estimated time is, 3.009697722005862
1930000
Estimated time is, 3.010171234077368
1940000
Estimated time is, 3.012298885373488
1950000
Estimated time is, 3.012757529343486
1960000
Estimated time is, 3.012926754431128
1970000
Estimated time is, 3.0140202218950676
1980000
Estimated time is, 3.013584711496395
1990000
Estimated time is, 3.0126758524424595
2000000
Estimated time is, 3.0126475702417137
2010000
Estimated time is, 3.0117925923321636
2020000
Estimated time is, 3.0112633862649814
2030000
Estimated time is, 3.0114495851582683
2040000
Estimated time is, 3.0109834317061708
2050000
Estimated time is, 3.054575539681315
2060000
Estimated time is, 3.05600608115592
2070000
Estimated time is, 3.0558971727167683
2080000
Estimated time is, 3.057672630359109
2090000
Estimated time is, 3.0586593535241806
2100000
Estimated time is, 3.05912968761976
2110000
Estimated time is, 3.059885298063356
2120000
Estimated time is, 3.0594343613934467
2130000
Estimated time is, 3.0593731884682644
2140000
Estimated time is, 3.0591775087485655
2150000
Estimated time is, 3.0590370627046233
2160000
Estimated time is, 3.0584815721660332
2170000
Estimated time is, 3.0579347568438195
2180000
Estimated time is, 3.058475174801778
2190000
Estimated time is, 3.0592110577652396
2200000
Estimated time is, 3.0591058046776975
2210000
Estimated time is, 3.0584728007949993
2220000
Estimated time is, 3.060775062875087
2230000
Estimated time is, 3.060146673885695
2240000
Estimated time is, 3.0593281621152117
2250000
Estimated time is, 3.0592938687290285
2260000
Estimated time is, 3.1040927803555785
2270000
Estimated time is, 3.105786997774323
2280000
Estimated time is, 3.1053030100911863
2290000
Estimated time is, 3.106206462905445
2300000
Estimated time is, 3.1057874779385743
2310000
Estimated time is, 3.1054256993082396
2320000
Estimated time is, 3.105596491429916
2330000
Estimated time is, 3.105420501898676
2340000
Estimated time is, 3.1051415095817507
2350000
Estimated time is, 3.1048437050639293
2360000
Estimated time is, 3.104517951383677
2370000
Estimated time is, 3.103898970715463
2380000
Estimated time is, 3.1035067811836163
2390000
Estimated time is, 3.10417107361182
2400000
Estimated time is, 3.10426617845978
2410000
Estimated time is, 3.1049297259398263
2420000
Estimated time is, 3.1055180256292965
2430000
Estimated time is, 3.1058262923465176
2440000
Estimated time is, 3.10534400473585
2450000
Estimated time is, 3.1062796944695754
2460000
Estimated time is, 3.1067919264785626
2470000
Estimated time is, 3.1062702654846204
2480000
Estimated time is, 3.1522653250291475
2490000
Estimated time is, 3.1549407690328506
2500000
Estimated time is, 3.155787641727403
2510000
Estimated time is, 3.155684593792348
2520000
Estimated time is, 3.1561903508099
2530000
Estimated time is, 3.157198246693482
2540000
Estimated time is, 3.1579089129356417
2550000
Estimated time is, 3.1582511266767894
2560000
Estimated time is, 3.1581353877583354
2570000
Estimated time is, 3.1591666951844735
2580000
Estimated time is, 3.160269896420695
2590000
Estimated time is, 3.1604799507388583
2600000
Estimated time is, 3.1605008926493956
2610000
Estimated time is, 3.161766426027679
2620000
Estimated time is, 3.161534269491729
2630000
Estimated time is, 3.1627916460365606
2640000
Estimated time is, 3.1628426748196907
2650000
Estimated time is, 3.162469994143653
2660000
Estimated time is, 3.1622051040454346
2670000
Estimated time is, 3.1619279704389234
2680000
Estimated time is, 3.1627779355925063
2690000
Estimated time is, 3.1629064416917427
2700000
Estimated time is, 3.163809582500084
2710000
Estimated time is, 3.164574920824986
2720000
Estimated time is, 3.164597685194318
2730000
Estimated time is, 3.2128917042131055
2740000
Estimated time is, 3.214329939156756
2750000
Estimated time is, 3.2154444480197597
2760000
Estimated time is, 3.2160801311690568
2770000
Estimated time is, 3.216073406229951
2780000
Estimated time is, 3.217249728451548
2790000
Estimated time is, 3.218264741447106
2800000
Estimated time is, 3.218083778083031
2810000
Estimated time is, 3.2189821834297963
2820000
Estimated time is, 3.2198314196878433
2830000
Estimated time is, 3.2193715797902556
2840000
Estimated time is, 3.2207981132084744
2850000
Estimated time is, 3.2206930143549966
2860000
Estimated time is, 3.2202457381886176
2870000
Estimated time is, 3.220601043347741
2880000
Estimated time is, 3.2210412778414907
2890000
Estimated time is, 3.2215148797092232
2900000
Estimated time is, 3.2203389215309506
2910000
Estimated time is, 3.2199784025145255
2920000
Estimated time is, 3.219794269720599
2930000
Estimated time is, 3.2200284331378346
2940000
Estimated time is, 3.2201444917810287
2950000
Estimated time is, 3.2205749966228363
2960000
Estimated time is, 3.2205167534301915
2970000
Estimated time is, 3.2209232835013886
2980000
Estimated time is, 3.221110306050543
2990000
Estimated time is, 3.2215758492260296
3000000
Estimated time is, 3.269165196360149
3010000
Estimated time is, 3.2702072809169476
3020000
Estimated time is, 3.2710163438876463
3030000
Estimated time is, 3.272337036140886
3040000
Estimated time is, 3.2725854947040167
3050000
Estimated time is, 3.2724313368703624
3060000
Estimated time is, 3.2736853830977717
3070000
Estimated time is, 3.2747837496062506
3080000
Estimated time is, 3.2752772165461526
3090000
Estimated time is, 3.276658631131692
3100000
Estimated time is, 3.27810549645177
3110000
Estimated time is, 3.2786049052497046
3120000
Estimated time is, 3.278990719308254
3130000
Estimated time is, 3.279735284468475
3140000
Estimated time is, 3.2798411921270936
3150000
Estimated time is, 3.280296866058187
3160000
Estimated time is, 3.280667130264479
3170000
Estimated time is, 3.281334913770074
3180000
Estimated time is, 3.2825329973863613
3190000
Estimated time is, 3.2824875268066624
3200000
Estimated time is, 3.283704449236559
3210000
Estimated time is, 3.2844216981185532
3220000
Estimated time is, 3.2856042315428535
3230000
Estimated time is, 3.285970389740189
3240000
Estimated time is, 3.2869869708143984
3250000
Estimated time is, 3.286442383508052
3260000
Estimated time is, 3.286522670903401
3270000
Estimated time is, 3.2868096791450574
3280000
Estimated time is, 3.2873805777216503
3290000
Estimated time is, 3.288652796405337
3300000
Estimated time is, 3.3397441376367403
3310000
Estimated time is, 3.341066660788399
3320000
Estimated time is, 3.3421780791043516
3330000
Estimated time is, 3.3429563538992357
3340000
Estimated time is, 3.344664724868584
3350000
Estimated time is, 3.347299399531171
3360000
Estimated time is, 3.348124412882689
3370000
Estimated time is, 3.349792041920743
3380000
Estimated time is, 3.351235344899535
3390000
Estimated time is, 3.3518226634413555
3400000
Estimated time is, 3.3552366593451133
3410000
Estimated time is, 3.3570072956911248
3420000
Estimated time is, 3.3578161581183767
3430000
Estimated time is, 3.3597642871708
3440000
Estimated time is, 3.36070305687233
3450000
Estimated time is, 3.3612116727517227
3460000
Estimated time is, 3.362080512997779
3470000
Estimated time is, 3.3625956159065673
3480000
Estimated time is, 3.3633581373435177
3490000
Estimated time is, 3.3645244346214565
3500000
Estimated time is, 3.364890601050398
3510000
Estimated time is, 3.3658546063982273
3520000
Estimated time is, 3.367243152533096
3530000
Estimated time is, 3.367518925493283
3540000
Estimated time is, 3.369267416656997
3550000
Estimated time is, 3.3702102506782907
3560000
Estimated time is, 3.370922338631114
3570000
Estimated time is, 3.3719051411094294
3580000
Estimated time is, 3.373564366405504
3590000
Estimated time is, 3.3737209512491937
3600000
Estimated time is, 3.3739938876137194
3610000
Estimated time is, 3.426812117578968
3620000
Estimated time is, 3.428097382166755
3630000
Estimated time is, 3.430122526033096
3640000
Estimated time is, 3.4302658454590134
3650000
Estimated time is, 3.431192667153239
3660000
Estimated time is, 3.432041976055352
3670000
Estimated time is, 3.432648622658427
3680000
Estimated time is, 3.434155316940502
3690000
Estimated time is, 3.4349747966697923
3700000
Estimated time is, 3.4354873824289793
3710000
Estimated time is, 3.4366275164934197
3720000
Estimated time is, 3.4375070094702833
3730000
Estimated time is, 3.437271218979663
3740000
Estimated time is, 3.4376296753570776
3750000
Estimated time is, 3.4390143445114414
3760000
Estimated time is, 3.4387275295010387
3770000
Estimated time is, 3.4395905172832086
3780000
Estimated time is, 3.4401259796558263
3790000
Estimated time is, 3.440828334365044
3800000
Estimated time is, 3.441890524750906
3810000
Estimated time is, 3.4432736303088234
3820000
Estimated time is, 3.443241052447358
3830000
Estimated time is, 3.4444847447906644
3840000
Estimated time is, 3.445386889512905
3850000
Estimated time is, 3.445327076897894
3860000
Estimated time is, 3.446880845275293
3870000
Estimated time is, 3.4476837777564184
3880000
Estimated time is, 3.4485803191466733
3890000
Estimated time is, 3.4497221837536243
3900000
Estimated time is, 3.450693606792218
3910000
Estimated time is, 3.4517588836972752
3920000
Estimated time is, 3.4526068855448333
3930000
Estimated time is, 3.453487036847512
3940000
Estimated time is, 3.4547262130025507
3950000
Estimated time is, 3.455699987513142
3960000
Estimated time is, 3.4570518965828154
3970000
Estimated time is, 3.5118163154626147
3980000
Estimated time is, 3.5134315312915074
3990000
Estimated time is, 3.5147730847647356
4000000
Estimated time is, 3.5164989219372855
4010000
Estimated time is, 3.5172678309946748
4020000
Estimated time is, 3.5180434602147908
4030000
Estimated time is, 3.5187189730114263
4040000
Estimated time is, 3.5192670325640627
4050000
Estimated time is, 3.5202870440824863
4060000
Estimated time is, 3.5210881540780843
4070000
Estimated time is, 3.521674179818627
4080000
Estimated time is, 3.5222005815210053
4090000
Estimated time is, 3.52235505240021
4100000
Estimated time is, 3.5228785270810565
4110000
Estimated time is, 3.5233467681233357

In [2]:

    

import pickle
pickle.dump(ph, open('postorder_hash_file_equivalence.pkl','wb'))


    

In [ ]: