In [1]:

    

import json
import datetime
# Necessary imports 
import sys
home_directory = '/dfs/scratch2/fcipollone'
sys.path.append(home_directory)
import os
import time
from nbminer.notebook_miner import NotebookMinerString
from nbminer.cells.cells import Cell
from nbminer.features.features import Features
from nbminer.stats.summary import Summary
from nbminer.stats.multiple_summary import MultipleSummary
from nbminer.encoders.ast_graph.ast_graph import *
from nbminer.pipeline.pipeline import Pipeline
from nbminer.features.features import Features
from nbminer.preprocess.get_ast_features import GetASTFeatures
from nbminer.preprocess.get_imports import GetImports
from nbminer.preprocess.get_docstrings import GetDocstrings
from nbminer.preprocess.resample_by_node import ResampleByNode
from nbminer.encoders.ast_graph.postorder_hash import PostorderHash, EquivalenceHash, LoopingHash
from nbminer.encoders.ast_graph.function_hash import FunctionHash
from lib2to3 import refactor


notebooks = []
filename = '/dfs/scratch2/fcipollone/stackoverflow/guesslang_and_ast/Posts_Reduced.xml'
output_file = open('/dfs/scratch2/fcipollone/stackoverflow/guesslang_and_ast/Posts_LoopingHash.xml','w')

count = 0

gastf = GetASTFeatures()
rbn = ResampleByNode()
gi = GetImports()
ph = LoopingHash(None, split_call=True, hash_many=False)
#pipe = Pipeline([gastf, rbn, gi, ph])
pipe = Pipeline([gastf, rbn, gi, ph])
start = time.time()

myfixes = ['lib2to3.fixes.fix_' + el for el in (refactor.get_all_fix_names("lib2to3.fixes"))]
refac = refactor.RefactoringTool(myfixes)
total_posts = 0
total_true_true_posts = 0
total_code_blocks = 0
total_true_true_code_blocks = 0
total_lines_of_code = 0
total_workable_codeblocks = 0
#4110319 total lines
for line in open(filename):
    total_posts += 1
    line_obj = json.loads(line)
    true_true_post = False
    new_code_block = False

    for el in line_obj['CodeBlocks']:
        total_code_blocks += 1
        code = el['code']
        if el['Guesslang'].lower().strip() != 'python' or el['Parsable'] != 'True':
            continue
        if not true_true_post:
            true_true_post = True
            total_true_true_posts += 1
        total_true_true_code_blocks += 1
        #### Determine if this is python two or three
        parsable = True
        try:
            ast.parse(code)
        except:
            try:
                tree = refac.refactor_string(code + '\n', 'cell')
                new_code = str(tree)[:-1]
                ast.parse(new_code)
                code = new_code
            except:
                parsable = False
        hashes = []
        imports = {}
        functions = []
        if parsable:
            total_workable_codeblocks += 1
            #total_through_pipe += 1
            nb = NotebookMinerString([code])
            a = Features([nb])
            a = pipe.transform(a)
            imports = a.get_notebook(0).get_feature('imports')
            functions = []
            for seg in a.get_list_segments():
                functions.extend(seg.get_feature('short_name_string'))
            total_lines_of_code += a.get_notebook(0).get_number_cells()
            for seg in a.get_notebook(0).cell_array:
                hashes.append(seg.get_feature('hash'))
        el['hashes'] = hashes
        el['imports'] = imports
        el['functions'] = functions
        new_code_block = True
    #if not new_code_block:
    #    continue
    
    json.dump(line_obj, output_file)
    output_file.write('\n')        
    count += 1
    if count % 10000 == 0:
        end = time.time()
        print (count)
        #print("Time so far per AST: ",(end-start)/total_through_pipe)
        #print("Time so far per post: ",(end-start)/count)
        print ("Estimated time is,",(end-start)/(count/4110319)/3600)


    

    




10000
Estimated time is, 2.296990566697677
20000
Estimated time is, 2.2970185913010073
30000
Estimated time is, 2.266414803467828
40000
Estimated time is, 2.3564723244998667
50000
Estimated time is, 2.335486631796975
60000
Estimated time is, 2.351129118785282
70000
Estimated time is, 2.378802633898422
80000
Estimated time is, 2.411255401631721
90000
Estimated time is, 2.418371168421961
100000
Estimated time is, 2.4123160151345004
110000
Estimated time is, 2.4422352791301596
120000
Estimated time is, 2.434849390942303
130000
Estimated time is, 2.4437848754036082
140000
Estimated time is, 2.4222157232744523
150000
Estimated time is, 2.4334914326325374
160000
Estimated time is, 2.408519103317517
170000
Estimated time is, 2.4251447454799253
180000
Estimated time is, 2.4100495429207474
190000
Estimated time is, 2.414982384824607
200000
Estimated time is, 2.394916035720238
210000
Estimated time is, 2.4189959336331133
220000
Estimated time is, 2.402838009901292
230000
Estimated time is, 2.438812418624622
240000
Estimated time is, 2.430533302592179
250000
Estimated time is, 2.450883741553622
260000
Estimated time is, 2.441757979503922
270000
Estimated time is, 2.43431346104692
280000
Estimated time is, 2.449372219163644
290000
Estimated time is, 2.4316388690712176
300000
Estimated time is, 2.416269973162709
310000
Estimated time is, 2.4289783995812755
320000
Estimated time is, 2.419038079991493
330000
Estimated time is, 2.406749014936282
340000
Estimated time is, 2.40045174090829
350000
Estimated time is, 2.415888540934948
360000
Estimated time is, 2.402843515463238
370000
Estimated time is, 2.391112216136316
380000
Estimated time is, 2.383491661504287
390000
Estimated time is, 2.40563168464805
400000
Estimated time is, 2.3951911552519176
410000
Estimated time is, 2.382987924989842
420000
Estimated time is, 2.3781814908840473
430000
Estimated time is, 2.3688808281313616
440000
Estimated time is, 2.3969780324315506
450000
Estimated time is, 2.3907623823930093
460000
Estimated time is, 2.384171528614334
470000
Estimated time is, 2.3806011374355016
480000
Estimated time is, 2.374843350294676
490000
Estimated time is, 2.397430713368022
500000
Estimated time is, 2.386644130471826
510000
Estimated time is, 2.3873230969473296
520000
Estimated time is, 2.381135521301559
530000
Estimated time is, 2.375623643675263
540000
Estimated time is, 2.4018410920929623
550000
Estimated time is, 2.3979197265386984
560000
Estimated time is, 2.397973340378464
570000
Estimated time is, 2.3939887144594185
580000
Estimated time is, 2.3909973148529424
590000
Estimated time is, 2.3866889993102687
600000
Estimated time is, 2.414900460658788
610000
Estimated time is, 2.413615524106083
620000
Estimated time is, 2.409077926306911
630000
Estimated time is, 2.4072562763479266
640000
Estimated time is, 2.408652265269057
650000
Estimated time is, 2.403903987623265
660000
Estimated time is, 2.4013024037683603
670000
Estimated time is, 2.4306663875810015
680000
Estimated time is, 2.4307502938433228
690000
Estimated time is, 2.4276923153582346
700000
Estimated time is, 2.4281969156073093
710000
Estimated time is, 2.427600455777202
720000
Estimated time is, 2.427720475757076
730000
Estimated time is, 2.423379921645443
740000
Estimated time is, 2.4548958064642967
750000
Estimated time is, 2.4554394693439288
760000
Estimated time is, 2.4590885017837527
770000
Estimated time is, 2.4561922648033896
780000
Estimated time is, 2.4545314583618834
790000
Estimated time is, 2.454462150286189
800000
Estimated time is, 2.4537783538328015
810000
Estimated time is, 2.4572024999876283
820000
Estimated time is, 2.489282108966468
830000
Estimated time is, 2.4911088617703494
840000
Estimated time is, 2.4916802807817073
850000
Estimated time is, 2.488928250109739
860000
Estimated time is, 2.4898274060948293
870000
Estimated time is, 2.488127258796039
880000
Estimated time is, 2.488389056386471
890000
Estimated time is, 2.487612154960724
900000
Estimated time is, 2.521110937637884
910000
Estimated time is, 2.5194818763095816
920000
Estimated time is, 2.522212834803856
930000
Estimated time is, 2.5238698279306853
940000
Estimated time is, 2.525277434666782
950000
Estimated time is, 2.5249502082111155
960000
Estimated time is, 2.5249447238652767
970000
Estimated time is, 2.52612921347068
980000
Estimated time is, 2.5304883481130194
990000
Estimated time is, 2.5633193183787983
1000000
Estimated time is, 2.5613559526119336
1010000
Estimated time is, 2.562431710068131
1020000
Estimated time is, 2.563358327191768
1030000
Estimated time is, 2.563533051957967
1040000
Estimated time is, 2.565478831712095
1050000
Estimated time is, 2.565158669635503
1060000
Estimated time is, 2.563434367170049
1070000
Estimated time is, 2.5619200031530274
1080000
Estimated time is, 2.562371198620736
1090000
Estimated time is, 2.597376566577414
1100000
Estimated time is, 2.598635772347363
1110000
Estimated time is, 2.5976464304953675
1120000
Estimated time is, 2.5978153255129035
1130000
Estimated time is, 2.5959008314038625
1140000
Estimated time is, 2.5966851940376947
1150000
Estimated time is, 2.5961109817468686
1160000
Estimated time is, 2.595947705538552
1170000
Estimated time is, 2.597389898748129
1180000
Estimated time is, 2.59812411514717
1190000
Estimated time is, 2.5961336538153317
1200000
Estimated time is, 2.6315649397211454
1210000
Estimated time is, 2.631077888527318
1220000
Estimated time is, 2.632691603585994
1230000
Estimated time is, 2.633998415059502
1240000
Estimated time is, 2.633958329527868
1250000
Estimated time is, 2.636382251862947
1260000
Estimated time is, 2.6369754522186852
1270000
Estimated time is, 2.638746594918346
1280000
Estimated time is, 2.6371720043691975
1290000
Estimated time is, 2.637951083990817
1300000
Estimated time is, 2.6378971340571953
1310000
Estimated time is, 2.6737509965575312
1320000
Estimated time is, 2.6737121583645584
1330000
Estimated time is, 2.6732917461502144
1340000
Estimated time is, 2.674010150668097
1350000
Estimated time is, 2.67595059968744
1360000
Estimated time is, 2.67511883232894
1370000
Estimated time is, 2.6748711732187096
1380000
Estimated time is, 2.673493002584799
1390000
Estimated time is, 2.6734205547184127
1400000
Estimated time is, 2.674756823693911
1410000
Estimated time is, 2.6745185985360433
1420000
Estimated time is, 2.6756925636142928
1430000
Estimated time is, 2.711234061625717
1440000
Estimated time is, 2.713171012384809
1450000
Estimated time is, 2.7128119189507216
1460000
Estimated time is, 2.7145642641332346
1470000
Estimated time is, 2.7146302428954483
1480000
Estimated time is, 2.7158344113780806
1490000
Estimated time is, 2.71524488400192
1500000
Estimated time is, 2.7173787849927677
1510000
Estimated time is, 2.7174577691521065
1520000
Estimated time is, 2.719163777054132
1530000
Estimated time is, 2.716662847195406
1540000
Estimated time is, 2.7143696603535643
1550000
Estimated time is, 2.715687014158258
1560000
Estimated time is, 2.713707775373568
1570000
Estimated time is, 2.713817777119373
1580000
Estimated time is, 2.7503076614675592
1590000
Estimated time is, 2.750212515675779
1600000
Estimated time is, 2.7509036186532296
1610000
Estimated time is, 2.7528091665112133
1620000
Estimated time is, 2.752644334259345
1630000
Estimated time is, 2.753521298504481
1640000
Estimated time is, 2.752617328972964
1650000
Estimated time is, 2.7522583341731908
1660000
Estimated time is, 2.752827125446624
1670000
Estimated time is, 2.7522377321552236
1680000
Estimated time is, 2.7516747261881487
1690000
Estimated time is, 2.7495245046891674
1700000
Estimated time is, 2.7498939598408283
1710000
Estimated time is, 2.7499332808532584
1720000
Estimated time is, 2.7484433119579217
1730000
Estimated time is, 2.746888830424771
1740000
Estimated time is, 2.7837711536961383
1750000
Estimated time is, 2.7845775981909737
1760000
Estimated time is, 2.7842030918516674
1770000
Estimated time is, 2.7845861665560148
1780000
Estimated time is, 2.785051110591182
1790000
Estimated time is, 2.786391626298847
1800000
Estimated time is, 2.7864416533733496
1810000
Estimated time is, 2.785318333367319
1820000
Estimated time is, 2.786164852500248
1830000
Estimated time is, 2.785154612826552
1840000
Estimated time is, 2.7857891364057625
1850000
Estimated time is, 2.78585385149435
1860000
Estimated time is, 2.788453019510104
1870000
Estimated time is, 2.7878826346249688
1880000
Estimated time is, 2.78717025536161
1890000
Estimated time is, 2.7867346816031495
1900000
Estimated time is, 2.786703152566072
1910000
Estimated time is, 2.7863792843465447
1920000
Estimated time is, 2.8256568460741156
1930000
Estimated time is, 2.8274627325489576
1940000
Estimated time is, 2.828437480235339
1950000
Estimated time is, 2.8277792052961934
1960000
Estimated time is, 2.8283009717107377
1970000
Estimated time is, 2.828366823058824
1980000
Estimated time is, 2.828780365698209
1990000
Estimated time is, 2.8290215678499795
2000000
Estimated time is, 2.829407360839646
2010000
Estimated time is, 2.828617284020893
2020000
Estimated time is, 2.8292616298729176
2030000
Estimated time is, 2.829484484109248
2040000
Estimated time is, 2.83121563529118
2050000
Estimated time is, 2.8313692712070018
2060000
Estimated time is, 2.8317210761923413
2070000
Estimated time is, 2.8313481000471095
2080000
Estimated time is, 2.832384498550375
2090000
Estimated time is, 2.832850220161957
2100000
Estimated time is, 2.8321198056205077
2110000
Estimated time is, 2.8724613639484824
2120000
Estimated time is, 2.8721992927633555
2130000
Estimated time is, 2.8724091929008435
2140000
Estimated time is, 2.8722305922253706
2150000
Estimated time is, 2.8731239679992537
2160000
Estimated time is, 2.8737742197214784
2170000
Estimated time is, 2.873769138967442
2180000
Estimated time is, 2.8749056850714276
2190000
Estimated time is, 2.8760044173912753
2200000
Estimated time is, 2.8758377546529736
2210000
Estimated time is, 2.876819600119769
2220000
Estimated time is, 2.8779286209074466
2230000
Estimated time is, 2.8772353388418095
2240000
Estimated time is, 2.8767606006597473
2250000
Estimated time is, 2.8768999768651287
2260000
Estimated time is, 2.876250578878784
2270000
Estimated time is, 2.876557333042996
2280000
Estimated time is, 2.87643734310705
2290000
Estimated time is, 2.8767458187114694
2300000
Estimated time is, 2.8754249316688862
2310000
Estimated time is, 2.874414255253029
2320000
Estimated time is, 2.9151166502700976
2330000
Estimated time is, 2.9161262483162016
2340000
Estimated time is, 2.9155863904196337
2350000
Estimated time is, 2.916434419254686
2360000
Estimated time is, 2.9168037204224135
2370000
Estimated time is, 2.916538101626034
2380000
Estimated time is, 2.9173873230460257
2390000
Estimated time is, 2.91680961596936
2400000
Estimated time is, 2.9172152891165215
2410000
Estimated time is, 2.9184255017477247
2420000
Estimated time is, 2.9202446745896418
2430000
Estimated time is, 2.9205881129861937
2440000
Estimated time is, 2.9201593145919533
2450000
Estimated time is, 2.9207829000167345
2460000
Estimated time is, 2.921651832480572
2470000
Estimated time is, 2.92047397834786
2480000
Estimated time is, 2.9195377474882735
2490000
Estimated time is, 2.919463360403638
2500000
Estimated time is, 2.9192942311923127
2510000
Estimated time is, 2.9192837337782174
2520000
Estimated time is, 2.9210331597470054
2530000
Estimated time is, 2.9213888135756374
2540000
Estimated time is, 2.9210674987213143
2550000
Estimated time is, 2.96263959537549
2560000
Estimated time is, 2.9631989917977912
2570000
Estimated time is, 2.965054828434272
2580000
Estimated time is, 2.9659259305295804
2590000
Estimated time is, 2.9669027871646847
2600000
Estimated time is, 2.967363387220906
2610000
Estimated time is, 2.9684539695570447
2620000
Estimated time is, 2.9709797948524748
2630000
Estimated time is, 2.9714270439760675
2640000
Estimated time is, 2.9711427134930934
2650000
Estimated time is, 2.9709186322569434
2660000
Estimated time is, 2.9723506833795574
2670000
Estimated time is, 2.9731378344271797
2680000
Estimated time is, 2.9740144575960685
2690000
Estimated time is, 2.9733548037445074
2700000
Estimated time is, 2.9731214261773156
2710000
Estimated time is, 2.9742654841657363
2720000
Estimated time is, 2.9735907169091607
2730000
Estimated time is, 2.974499414144405
2740000
Estimated time is, 2.9750704461163404
2750000
Estimated time is, 2.974984627179911
2760000
Estimated time is, 2.9746611358748134
2770000
Estimated time is, 2.9744437828631876
2780000
Estimated time is, 2.9750549685941343
2790000
Estimated time is, 2.9758669225029335
2800000
Estimated time is, 3.018308833349069
2810000
Estimated time is, 3.0202652556683285
2820000
Estimated time is, 3.020593279440936
2830000
Estimated time is, 3.020839221600403
2840000
Estimated time is, 3.021548648737989
2850000
Estimated time is, 3.0211262843569178
2860000
Estimated time is, 3.0211613651427744
2870000
Estimated time is, 3.0212450506128117
2880000
Estimated time is, 3.0209305324332076
2890000
Estimated time is, 3.0217818062179136
2900000
Estimated time is, 3.0213088858319805
2910000
Estimated time is, 3.020829595880672
2920000
Estimated time is, 3.0208477413787107
2930000
Estimated time is, 3.021545270933485
2940000
Estimated time is, 3.0212677369819216
2950000
Estimated time is, 3.020782829015935
2960000
Estimated time is, 3.0206106280431975
2970000
Estimated time is, 3.0212797024346987
2980000
Estimated time is, 3.0219307433903673
2990000
Estimated time is, 3.0221235035526837
3000000
Estimated time is, 3.0211924100373517
3010000
Estimated time is, 3.0208656118447546
3020000
Estimated time is, 3.020968533692682
3030000
Estimated time is, 3.0215538971944924
3040000
Estimated time is, 3.020995688005244
3050000
Estimated time is, 3.0202801300607938
3060000
Estimated time is, 3.020276689545507
3070000
Estimated time is, 3.0205357450747634
3080000
Estimated time is, 3.0204604227072522
3090000
Estimated time is, 3.0666954714966783
3100000
Estimated time is, 3.069090989200925
3110000
Estimated time is, 3.069987916269535
3120000
Estimated time is, 3.070402131990181
3130000
Estimated time is, 3.0720475643269767
3140000
Estimated time is, 3.0725294407030646
3150000
Estimated time is, 3.073198570954128
3160000
Estimated time is, 3.0746364578344116
3170000
Estimated time is, 3.0760638393184756
3180000
Estimated time is, 3.076674935428242
3190000
Estimated time is, 3.0774426841526057
3200000
Estimated time is, 3.077950990638266
3210000
Estimated time is, 3.0782132170520593
3220000
Estimated time is, 3.078180412564402
3230000
Estimated time is, 3.0784029693993302
3240000
Estimated time is, 3.079096880816635
3250000
Estimated time is, 3.0782808870824394
3260000
Estimated time is, 3.0785873260698775
3270000
Estimated time is, 3.0782762888084094
3280000
Estimated time is, 3.078958121712381
3290000
Estimated time is, 3.0795720036232974
3300000
Estimated time is, 3.079626422557208
3310000
Estimated time is, 3.079395719900634
3320000
Estimated time is, 3.079606488173307
3330000
Estimated time is, 3.0802159793726642
3340000
Estimated time is, 3.080191265515397
3350000
Estimated time is, 3.0821204553497834
3360000
Estimated time is, 3.0827957292426076
3370000
Estimated time is, 3.0833725899688886
3380000
Estimated time is, 3.0836939707611157
3390000
Estimated time is, 3.1300730902091485
3400000
Estimated time is, 3.132838794623103
3410000
Estimated time is, 3.134107157779355
3420000
Estimated time is, 3.1358351808865494
3430000
Estimated time is, 3.137583835969027
3440000
Estimated time is, 3.1395333397888896
3450000
Estimated time is, 3.14004520003571
3460000
Estimated time is, 3.1414039630474493
3470000
Estimated time is, 3.1417117118741995
3480000
Estimated time is, 3.1433787734352716
3490000
Estimated time is, 3.1442910970854805
3500000
Estimated time is, 3.1451096641774834
3510000
Estimated time is, 3.1459442203284196
3520000
Estimated time is, 3.1466690376187065
3530000
Estimated time is, 3.147581389824369
3540000
Estimated time is, 3.1488990337482643
3550000
Estimated time is, 3.1497507995816227
3560000
Estimated time is, 3.149911094314537
3570000
Estimated time is, 3.1512553466512414
3580000
Estimated time is, 3.1525985512029884
3590000
Estimated time is, 3.1525773462994775
3600000
Estimated time is, 3.152794574795869
3610000
Estimated time is, 3.1539140448636807
3620000
Estimated time is, 3.154007720124623
3630000
Estimated time is, 3.15529786901661
3640000
Estimated time is, 3.1549791708143586
3650000
Estimated time is, 3.1552552957081126
3660000
Estimated time is, 3.1554975838060813
3670000
Estimated time is, 3.1555427112718206
3680000
Estimated time is, 3.1561815313098265
3690000
Estimated time is, 3.1565486434552033
3700000
Estimated time is, 3.1567684371767193
3710000
Estimated time is, 3.1572212809343374
3720000
Estimated time is, 3.205074209962147
3730000
Estimated time is, 3.206217574568974
3740000
Estimated time is, 3.2070487351450527
3750000
Estimated time is, 3.208828362314223
3760000
Estimated time is, 3.20822345251154
3770000
Estimated time is, 3.2095298260080565
3780000
Estimated time is, 3.2103368288024368
3790000
Estimated time is, 3.211299204873135
3800000
Estimated time is, 3.2119751645060544
3810000
Estimated time is, 3.2134193445888948
3820000
Estimated time is, 3.213669756440217
3830000
Estimated time is, 3.2146380415250824
3840000
Estimated time is, 3.2153205710431556
3850000
Estimated time is, 3.2152695903895454
3860000
Estimated time is, 3.2163938660298776
3870000
Estimated time is, 3.2171747002532305
3880000
Estimated time is, 3.217565019221539
3890000
Estimated time is, 3.2183470915865695
3900000
Estimated time is, 3.2186682570492167
3910000
Estimated time is, 3.2195487058018064
3920000
Estimated time is, 3.220151187314337
3930000
Estimated time is, 3.2200750904324584
3940000
Estimated time is, 3.2209389324940183
3950000
Estimated time is, 3.2213343834928887
3960000
Estimated time is, 3.222678942321079
3970000
Estimated time is, 3.223393931696296
3980000
Estimated time is, 3.223875937212208
3990000
Estimated time is, 3.2241891787339187
4000000
Estimated time is, 3.224674635035208
4010000
Estimated time is, 3.2246041898691016
4020000
Estimated time is, 3.224821488564011
4030000
Estimated time is, 3.2256884516278883
4040000
Estimated time is, 3.2258578082804013
4050000
Estimated time is, 3.2265376448757994
4060000
Estimated time is, 3.2270879765834586
4070000
Estimated time is, 3.22791641555822
4080000
Estimated time is, 3.227913795810574
4090000
Estimated time is, 3.228243895991381
4100000
Estimated time is, 3.278046668227236
4110000
Estimated time is, 3.279316013021674

In [2]:

    

import pickle
pickle.dump(ph, open('postorder_hash_file_loop_hash.pkl','wb'))


    

In [ ]: