In [2]:

    

import json
import datetime
# Necessary imports 
import sys
home_directory = '/dfs/scratch2/fcipollone'
sys.path.append(home_directory)
import os
import time
from nbminer.notebook_miner import NotebookMinerString
from nbminer.cells.cells import Cell
from nbminer.features.features import Features
from nbminer.stats.summary import Summary
from nbminer.stats.multiple_summary import MultipleSummary
from nbminer.encoders.ast_graph.ast_graph import *
from nbminer.pipeline.pipeline import Pipeline
from nbminer.features.features import Features
from nbminer.preprocess.get_ast_features import GetASTFeatures
from nbminer.preprocess.get_imports import GetImports
from nbminer.preprocess.get_docstrings import GetDocstrings
from nbminer.preprocess.resample_by_node import ResampleByNode
from nbminer.encoders.ast_graph.postorder_hash import PostorderHash, EquivalenceHash, LoopingHash, LoopingCollapseHash
from nbminer.encoders.ast_graph.function_hash import FunctionHash
from lib2to3 import refactor


notebooks = []
filename = '/dfs/scratch2/fcipollone/stackoverflow/guesslang_and_ast/Posts_Reduced.xml'
output_file = open('/dfs/scratch2/fcipollone/stackoverflow/guesslang_and_ast/Posts_LoopCollapseHash.xml','w')

count = 0

gastf = GetASTFeatures()
rbn = ResampleByNode()
gi = GetImports()
ph = LoopingCollapseHash(None, split_call=True)
#pipe = Pipeline([gastf, rbn, gi, ph])
pipe = Pipeline([gastf, rbn, gi, ph])
start = time.time()

myfixes = ['lib2to3.fixes.fix_' + el for el in (refactor.get_all_fix_names("lib2to3.fixes"))]
refac = refactor.RefactoringTool(myfixes)
total_posts = 0
total_true_true_posts = 0
total_code_blocks = 0
total_true_true_code_blocks = 0
total_lines_of_code = 0
total_workable_codeblocks = 0
#4110319 total lines
for line in open(filename):
    total_posts += 1
    line_obj = json.loads(line)
    true_true_post = False
    new_code_block = False

    for el in line_obj['CodeBlocks']:
        total_code_blocks += 1
        code = el['code']
        if el['Guesslang'].lower().strip() != 'python' or el['Parsable'] != 'True':
            continue
        if not true_true_post:
            true_true_post = True
            total_true_true_posts += 1
        total_true_true_code_blocks += 1
        #### Determine if this is python two or three
        parsable = True
        try:
            ast.parse(code)
        except:
            try:
                tree = refac.refactor_string(code + '\n', 'cell')
                new_code = str(tree)[:-1]
                ast.parse(new_code)
                code = new_code
            except:
                parsable = False
        hashes = []
        imports = {}
        functions = []
        if parsable:
            total_workable_codeblocks += 1
            #total_through_pipe += 1
            nb = NotebookMinerString([code])
            a = Features([nb])
            a = pipe.transform(a)
            imports = a.get_notebook(0).get_feature('imports')
            functions = []
            for seg in a.get_list_segments():
                functions.extend(seg.get_feature('short_name_string'))
            total_lines_of_code += a.get_notebook(0).get_number_cells()
            for seg in a.get_notebook(0).cell_array:
                hashes.append(seg.get_feature('hash'))
        el['hashes'] = hashes
        el['imports'] = imports
        el['functions'] = functions
        new_code_block = True
    #if not new_code_block:
    #    continue
    
    json.dump(line_obj, output_file)
    output_file.write('\n')        
    count += 1
    if count % 10000 == 0:
        end = time.time()
        print (count)
        #print("Time so far per AST: ",(end-start)/total_through_pipe)
        #print("Time so far per post: ",(end-start)/count)
        print ("Estimated time is,",(end-start)/(count/4110319)/3600)


    

    




10000
Estimated time is, 2.652904907282226
20000
Estimated time is, 2.589749782745341
30000
Estimated time is, 2.5473891679475926
40000
Estimated time is, 2.6626576597921843
50000
Estimated time is, 2.6101799826224235
60000
Estimated time is, 2.643086614312545
70000
Estimated time is, 2.6767810954494853
80000
Estimated time is, 2.705827250792199
90000
Estimated time is, 2.7876686083451343
100000
Estimated time is, 2.737608248678597
110000
Estimated time is, 2.7475821016856754
120000
Estimated time is, 2.737849936723181
130000
Estimated time is, 2.750667362881756
140000
Estimated time is, 2.765008026046247
150000
Estimated time is, 2.7406345955124176
160000
Estimated time is, 2.7434102990866176
170000
Estimated time is, 2.719596359980898
180000
Estimated time is, 2.7375014726865676
190000
Estimated time is, 2.701391974064149
200000
Estimated time is, 2.7130101709956294
210000
Estimated time is, 2.7196354965117946
220000
Estimated time is, 2.7379981587747175
230000
Estimated time is, 2.745455165306102
240000
Estimated time is, 2.7843956542270916
250000
Estimated time is, 2.782643734509615
260000
Estimated time is, 2.8127349941888253
270000
Estimated time is, 2.80080310218528
280000
Estimated time is, 2.7891173727689176
290000
Estimated time is, 2.8002320196775035
300000
Estimated time is, 2.783850360415088
310000
Estimated time is, 2.758854661428618
320000
Estimated time is, 2.7833692576670424
330000
Estimated time is, 2.768481115020912
340000
Estimated time is, 2.7549758944755838
350000
Estimated time is, 2.7355220484411866
360000
Estimated time is, 2.76090357490219
370000
Estimated time is, 2.7467386584843334
380000
Estimated time is, 2.7342472164023266
390000
Estimated time is, 2.7216498535893114
400000
Estimated time is, 2.743110012535614
410000
Estimated time is, 2.727120319189511
420000
Estimated time is, 2.719452445731957
430000
Estimated time is, 2.7135111676051813
440000
Estimated time is, 2.7413871337551736
450000
Estimated time is, 2.7355930808304683
460000
Estimated time is, 2.7298500966105133
470000
Estimated time is, 2.7242700327102845
480000
Estimated time is, 2.7491050446858605
490000
Estimated time is, 2.7385289871865326
500000
Estimated time is, 2.7303889317427674
510000
Estimated time is, 2.7273685668109535
520000
Estimated time is, 2.718319637043569
530000
Estimated time is, 2.748909964546023
540000
Estimated time is, 2.740539992904375
550000
Estimated time is, 2.7382373974969343
560000
Estimated time is, 2.7355818532519085
570000
Estimated time is, 2.7306397980926826
580000
Estimated time is, 2.757026371247842
590000
Estimated time is, 2.753319142080881
600000
Estimated time is, 2.747273314169224
610000
Estimated time is, 2.740889466225628
620000
Estimated time is, 2.7325538743722393
630000
Estimated time is, 2.72753811836229
640000
Estimated time is, 2.7625381992878193
650000
Estimated time is, 2.7592896615034936
660000
Estimated time is, 2.753588726480541
670000
Estimated time is, 2.7480279659862954
680000
Estimated time is, 2.744885739848704
690000
Estimated time is, 2.739588079250484
700000
Estimated time is, 2.766581024301175
710000
Estimated time is, 2.7636065566914283
720000
Estimated time is, 2.7590930733893115
730000
Estimated time is, 2.757233668206836
740000
Estimated time is, 2.756564304465244
750000
Estimated time is, 2.7580700498914488
760000
Estimated time is, 2.755150713654993
770000
Estimated time is, 2.7894687712060118
780000
Estimated time is, 2.788380936845452
790000
Estimated time is, 2.787017581098406
800000
Estimated time is, 2.7849472793219103
810000
Estimated time is, 2.785325781484041
820000
Estimated time is, 2.785462706847715
830000
Estimated time is, 2.8197592742319224
840000
Estimated time is, 2.8186083996506093
850000
Estimated time is, 2.8155436279544936
860000
Estimated time is, 2.8130424304771244
870000
Estimated time is, 2.8111504312927726
880000
Estimated time is, 2.8103850970012862
890000
Estimated time is, 2.8098378524222243
900000
Estimated time is, 2.8118246715978343
910000
Estimated time is, 2.848703521167791
920000
Estimated time is, 2.8501639359797295
930000
Estimated time is, 2.849810012162992
940000
Estimated time is, 2.8477223632998667
950000
Estimated time is, 2.8502061468099824
960000
Estimated time is, 2.8477201737555604
970000
Estimated time is, 2.846634370998418
980000
Estimated time is, 2.8471602081172547
990000
Estimated time is, 2.8816943293305894
1000000
Estimated time is, 2.8810838650961976
1010000
Estimated time is, 2.879967854992834
1020000
Estimated time is, 2.879028409191762
1030000
Estimated time is, 2.8777195783983993
1040000
Estimated time is, 2.877166715994244
1050000
Estimated time is, 2.8750021176036977
1060000
Estimated time is, 2.872916420808749
1070000
Estimated time is, 2.9085147680248964
1080000
Estimated time is, 2.9074555019991113
1090000
Estimated time is, 2.9106534704211184
1100000
Estimated time is, 2.911105109025965
1110000
Estimated time is, 2.9120795492526037
1120000
Estimated time is, 2.911552876759972
1130000
Estimated time is, 2.9103897616765795
1140000
Estimated time is, 2.90775522428099
1150000
Estimated time is, 2.9070013737106652
1160000
Estimated time is, 2.905512713139438
1170000
Estimated time is, 2.944971810940928
1180000
Estimated time is, 2.948489551283447
1190000
Estimated time is, 2.9491294217305097
1200000
Estimated time is, 2.949106657527269
1210000
Estimated time is, 2.9488365454572256
1220000
Estimated time is, 2.9513932223143904
1230000
Estimated time is, 2.9519427768964706
1240000
Estimated time is, 2.9535734073871374
1250000
Estimated time is, 2.953492995287123
1260000
Estimated time is, 2.996152416876673
1270000
Estimated time is, 3.0004707929867003
1280000
Estimated time is, 3.0015627211413776
1290000
Estimated time is, 3.0030049861750077
1300000
Estimated time is, 3.003927183194487
1310000
Estimated time is, 3.0026855825464613
1320000
Estimated time is, 3.0026058622776666
1330000
Estimated time is, 3.0012271571004643
1340000
Estimated time is, 3.000891223834636
1350000
Estimated time is, 2.999493372881827
1360000
Estimated time is, 2.9986846626384236
1370000
Estimated time is, 3.040245469150504
1380000
Estimated time is, 3.0401762422590823
1390000
Estimated time is, 3.0400072242712843
1400000
Estimated time is, 3.040740727413972
1410000
Estimated time is, 3.0410735489116907
1420000
Estimated time is, 3.041014081023167
1430000
Estimated time is, 3.039647983277115
1440000
Estimated time is, 3.040364496622761
1450000
Estimated time is, 3.0380837308751794
1460000
Estimated time is, 3.0374829326738286
1470000
Estimated time is, 3.0360146246721014
1480000
Estimated time is, 3.0784518770924367
1490000
Estimated time is, 3.079721816201489
1500000
Estimated time is, 3.0819815864970885
1510000
Estimated time is, 3.0826933816594346
1520000
Estimated time is, 3.083707953105069
1530000
Estimated time is, 3.0823158674513977
1540000
Estimated time is, 3.080067255921062
1550000
Estimated time is, 3.080890629163222
1560000
Estimated time is, 3.0793684850105905
1570000
Estimated time is, 3.0777901267077303
1580000
Estimated time is, 3.0758498137037993
1590000
Estimated time is, 3.075955236742407
1600000
Estimated time is, 3.0760724345459995
1610000
Estimated time is, 3.119676905203427
1620000
Estimated time is, 3.1223927858112095
1630000
Estimated time is, 3.122280672942809
1640000
Estimated time is, 3.1228064354312632
1650000
Estimated time is, 3.1238987703972003
1660000
Estimated time is, 3.1247961989242348
1670000
Estimated time is, 3.1236073947301133
1680000
Estimated time is, 3.121919577215002
1690000
Estimated time is, 3.1201873749997686
1700000
Estimated time is, 3.118444301275094
1710000
Estimated time is, 3.1168017108937804
1720000
Estimated time is, 3.1149430387220627
1730000
Estimated time is, 3.1138144731793167
1740000
Estimated time is, 3.114636675068626
1750000
Estimated time is, 3.1558879405402522
1760000
Estimated time is, 3.1560606975104166
1770000
Estimated time is, 3.157342667261622
1780000
Estimated time is, 3.159473071812115
1790000
Estimated time is, 3.159542177897859
1800000
Estimated time is, 3.159483705375251
1810000
Estimated time is, 3.157878920462998
1820000
Estimated time is, 3.157062727059844
1830000
Estimated time is, 3.1563939675492034
1840000
Estimated time is, 3.156807513214893
1850000
Estimated time is, 3.1562773563540563
1860000
Estimated time is, 3.156298030133535
1870000
Estimated time is, 3.1557697955801136
1880000
Estimated time is, 3.155241378439945
1890000
Estimated time is, 3.1557341333685645
1900000
Estimated time is, 3.197597190228932
1910000
Estimated time is, 3.1988989871631524
1920000
Estimated time is, 3.1996537097785986
1930000
Estimated time is, 3.2011080503156277
1940000
Estimated time is, 3.2021314578589535
1950000
Estimated time is, 3.20340177619276
1960000
Estimated time is, 3.204365114309385
1970000
Estimated time is, 3.2043451238105045
1980000
Estimated time is, 3.2039657098755403
1990000
Estimated time is, 3.203218755810564
2000000
Estimated time is, 3.204356440473688
2010000
Estimated time is, 3.204866636187944
2020000
Estimated time is, 3.2041638239584214
2030000
Estimated time is, 3.204687442078982
2040000
Estimated time is, 3.205368633331095
2050000
Estimated time is, 3.2057959492227805
2060000
Estimated time is, 3.250229655327306
2070000
Estimated time is, 3.2517811433050845
2080000
Estimated time is, 3.252738849776652
2090000
Estimated time is, 3.2540156994520815
2100000
Estimated time is, 3.253689461627258
2110000
Estimated time is, 3.252439715763031
2120000
Estimated time is, 3.250731352527695
2130000
Estimated time is, 3.2511308655789395
2140000
Estimated time is, 3.2507709847185255
2150000
Estimated time is, 3.2512578618319528
2160000
Estimated time is, 3.250810379017342
2170000
Estimated time is, 3.250607022613497
2180000
Estimated time is, 3.2501219081094566
2190000
Estimated time is, 3.251257127297119
2200000
Estimated time is, 3.2519887580627214
2210000
Estimated time is, 3.2515190496150184
2220000
Estimated time is, 3.251843545595094
2230000
Estimated time is, 3.2509184938752034
2240000
Estimated time is, 3.295872021507576
2250000
Estimated time is, 3.2980355055752546
2260000
Estimated time is, 3.2989614164200893
2270000
Estimated time is, 3.300238056322326
2280000
Estimated time is, 3.3006188150846465
2290000
Estimated time is, 3.300494270739568
2300000
Estimated time is, 3.29970021034403
2310000
Estimated time is, 3.2982906266599112
2320000
Estimated time is, 3.29732081803953
2330000
Estimated time is, 3.2964536360076235
2340000
Estimated time is, 3.2951573971586945
2350000
Estimated time is, 3.2941228402961253
2360000
Estimated time is, 3.294406131181964
2370000
Estimated time is, 3.293542245881543
2380000
Estimated time is, 3.292474170651436
2390000
Estimated time is, 3.2913617113648117
2400000
Estimated time is, 3.2913627784853605
2410000
Estimated time is, 3.2912542586473
2420000
Estimated time is, 3.291705153945427
2430000
Estimated time is, 3.3373787152078482
2440000
Estimated time is, 3.339414837004811
2450000
Estimated time is, 3.3410426513813216
2460000
Estimated time is, 3.342606710589862
2470000
Estimated time is, 3.3425940519711403
2480000
Estimated time is, 3.3423569932468946
2490000
Estimated time is, 3.3437888359959747
2500000
Estimated time is, 3.343294360104451
2510000
Estimated time is, 3.3424549645856754
2520000
Estimated time is, 3.3424022260534216
2530000
Estimated time is, 3.342466476322473
2540000
Estimated time is, 3.3419411603328135
2550000
Estimated time is, 3.3416090393888256
2560000
Estimated time is, 3.3405437131176314
2570000
Estimated time is, 3.340819310729491
2580000
Estimated time is, 3.3409849570053822
2590000
Estimated time is, 3.341362107056582
2600000
Estimated time is, 3.3414244106147457
2610000
Estimated time is, 3.342662974199752
2620000
Estimated time is, 3.3435962135504678
2630000
Estimated time is, 3.3436534939051024
2640000
Estimated time is, 3.391167557663375
2650000
Estimated time is, 3.391796347703326
2660000
Estimated time is, 3.3922638203899487
2670000
Estimated time is, 3.392428152812363
2680000
Estimated time is, 3.393261181538197
2690000
Estimated time is, 3.3931046009976433
2700000
Estimated time is, 3.393102917238587
2710000
Estimated time is, 3.393006375630544
2720000
Estimated time is, 3.392429308013748
2730000
Estimated time is, 3.3927507209074177
2740000
Estimated time is, 3.3930316675927514
2750000
Estimated time is, 3.393870296041806
2760000
Estimated time is, 3.3929421264312207
2770000
Estimated time is, 3.3915747822074405
2780000
Estimated time is, 3.3921481408938505
2790000
Estimated time is, 3.391653515904006
2800000
Estimated time is, 3.390595110741798
2810000
Estimated time is, 3.3906432258819383
2820000
Estimated time is, 3.3901148222988695
2830000
Estimated time is, 3.3891094840114215
2840000
Estimated time is, 3.3901567259151246
2850000
Estimated time is, 3.389149925114358
2860000
Estimated time is, 3.436224908923108
2870000
Estimated time is, 3.437736584387875
2880000
Estimated time is, 3.438449436014762
2890000
Estimated time is, 3.438621911273904
2900000
Estimated time is, 3.4379571025724633
2910000
Estimated time is, 3.437440844793713
2920000
Estimated time is, 3.438179532866633
2930000
Estimated time is, 3.439225717687794
2940000
Estimated time is, 3.4390176752483197
2950000
Estimated time is, 3.4385191595975786
2960000
Estimated time is, 3.4379846359311577
2970000
Estimated time is, 3.437332250584783
2980000
Estimated time is, 3.4369484128951004
2990000
Estimated time is, 3.43685084457934
3000000
Estimated time is, 3.4357193408454068
3010000
Estimated time is, 3.434945115466746
3020000
Estimated time is, 3.434995984371036
3030000
Estimated time is, 3.435255798981177
3040000
Estimated time is, 3.4344481321158384
3050000
Estimated time is, 3.433961717068697
3060000
Estimated time is, 3.4341176858561013
3070000
Estimated time is, 3.433741626830075
3080000
Estimated time is, 3.4338490942032265
3090000
Estimated time is, 3.4338639159457918
3100000
Estimated time is, 3.4839145008470758
3110000
Estimated time is, 3.485794137127623
3120000
Estimated time is, 3.486587521110991
3130000
Estimated time is, 3.4885345244217962
3140000
Estimated time is, 3.4892008173627107
3150000
Estimated time is, 3.489706187865467
3160000
Estimated time is, 3.4907127724440707
3170000
Estimated time is, 3.49180635621233
3180000
Estimated time is, 3.4922326137182864
3190000
Estimated time is, 3.4934544361463247
3200000
Estimated time is, 3.4939271813855335
3210000
Estimated time is, 3.4942292152471115
3220000
Estimated time is, 3.4952790751156813
3230000
Estimated time is, 3.4960548270841243
3240000
Estimated time is, 3.4967157962571815
3250000
Estimated time is, 3.4964804015933226
3260000
Estimated time is, 3.496667484207797
3270000
Estimated time is, 3.4962252621157335
3280000
Estimated time is, 3.497474426082978
3290000
Estimated time is, 3.498414043142365
3300000
Estimated time is, 3.499060719617358
3310000
Estimated time is, 3.49862221464678
3320000
Estimated time is, 3.499289918522243
3330000
Estimated time is, 3.4998120574811553
3340000
Estimated time is, 3.4997089673178485
3350000
Estimated time is, 3.501598542521128
3360000
Estimated time is, 3.552547344842636
3370000
Estimated time is, 3.5548409521886675
3380000
Estimated time is, 3.5563297028441663
3390000
Estimated time is, 3.5574959717008734
3400000
Estimated time is, 3.5597780332080116
3410000
Estimated time is, 3.5613503899278016
3420000
Estimated time is, 3.562466693715644
3430000
Estimated time is, 3.5646868248964574
3440000
Estimated time is, 3.5652908937301815
3450000
Estimated time is, 3.5657369776121683
3460000
Estimated time is, 3.5664709031079522
3470000
Estimated time is, 3.5663221670661236
3480000
Estimated time is, 3.5666334598487754
3490000
Estimated time is, 3.567191913459715
3500000
Estimated time is, 3.567329500703487
3510000
Estimated time is, 3.567515174675402
3520000
Estimated time is, 3.567661785027165
3530000
Estimated time is, 3.567884162050992
3540000
Estimated time is, 3.5691640904744997
3550000
Estimated time is, 3.56902488017466
3560000
Estimated time is, 3.5690015654628944
3570000
Estimated time is, 3.5695051704635175
3580000
Estimated time is, 3.570212645502454
3590000
Estimated time is, 3.569679533789137
3600000
Estimated time is, 3.5697231863887127
3610000
Estimated time is, 3.56997447317238
3620000
Estimated time is, 3.6202912240559892
3630000
Estimated time is, 3.6226319581339643
3640000
Estimated time is, 3.622900977390146
3650000
Estimated time is, 3.624137056006573
3660000
Estimated time is, 3.6241181561839966
3670000
Estimated time is, 3.6241655653983655
3680000
Estimated time is, 3.624568525471296
3690000
Estimated time is, 3.625086630564181
3700000
Estimated time is, 3.6252340248681443
3710000
Estimated time is, 3.6253688353071434
3720000
Estimated time is, 3.6258808125827366
3730000
Estimated time is, 3.625405188735594
3740000
Estimated time is, 3.6254232346773896
3750000
Estimated time is, 3.625919047660447
3760000
Estimated time is, 3.625005124899033
3770000
Estimated time is, 3.625865587709366
3780000
Estimated time is, 3.6261850846562034
3790000
Estimated time is, 3.62581176594335
3800000
Estimated time is, 3.6261477500881694
3810000
Estimated time is, 3.626784609917044
3820000
Estimated time is, 3.626578000237685
3830000
Estimated time is, 3.626847646789898
3840000
Estimated time is, 3.6270102248049185
3850000
Estimated time is, 3.626687007808553
3860000
Estimated time is, 3.6274082823507596
3870000
Estimated time is, 3.6275409375706857
3880000
Estimated time is, 3.627812117218466
3890000
Estimated time is, 3.6285039024161865
3900000
Estimated time is, 3.6287303156418074
3910000
Estimated time is, 3.6295258705806197
3920000
Estimated time is, 3.630185128220367
3930000
Estimated time is, 3.683625799487082
3940000
Estimated time is, 3.6855283521430167
3950000
Estimated time is, 3.687317044444354
3960000
Estimated time is, 3.688927503790066
3970000
Estimated time is, 3.6897784483425675
3980000
Estimated time is, 3.690254696444837
3990000
Estimated time is, 3.6907035026491477
4000000
Estimated time is, 3.690926729286177
4010000
Estimated time is, 3.6907392212566075
4020000
Estimated time is, 3.69044806483721
4030000
Estimated time is, 3.6907785209135855
4040000
Estimated time is, 3.6907983787752583
4050000
Estimated time is, 3.691312262577449
4060000
Estimated time is, 3.6918052153655685
4070000
Estimated time is, 3.6922688427131862
4080000
Estimated time is, 3.6924879274988354
4090000
Estimated time is, 3.692433614011931
4100000
Estimated time is, 3.6928969072351863
4110000
Estimated time is, 3.6930319969989434

In [4]:

    

import pickle
pickle.dump(ph, open('postorder_hash_file_loop_collapse_hash.pkl','wb'))


    

In [ ]: