In [5]:

    

import ujson
import gzip
import numpy as np
import math
from os.path import isfile, isdir
from struct import pack, unpack
from os import makedirs
import os
from struct import pack
import itertools
import scipy.sparse, scipy.io
import intbitset
import pickle as pickle


    

In [8]:

    

fname = 'C:\\vk\\group_data.bin'
invfname = 'C:\\vk\\group_data_inv.bin'


    

In [9]:

    

n_nodes = 350000000


    

In [10]:

    

memcount = np.zeros(n_nodes, dtype=np.int)
buffer_size = 1024
last_node = 1
cnt = 0
with open(fname, 'rb') as f:
    while True:
        bytes = f.read(4*buffer_size)
        if len(bytes) == 0:
            break
        buffer_values = unpack('i'*(len(bytes)/4), bytes)
        for idx, value in enumerate(buffer_values):
            if value <= 0:
                last_node = -value
                cnt += 1
                if cnt%100000==0:
                    print last_node, cnt
            else:
                memcount[value] += 1
        del(bytes)


    

    




113737 100000
228876 200000
344079 300000
463516 400000
583321 500000
698462 600000
815079 700000
931946 800000
1048433 900000
1165598 1000000
1282171 1100000
1413511 1200000
1529700 1300000
1645369 1400000
1763036 1500000
1880112 1600000
1996892 1700000
2113407 1800000
2230177 1900000
2345467 2000000
2461750 2100000
2585634 2200000
2740121 2300000
2989358 2400000
3211816 2500000
3381999 2600000
3518242 2700000
3639270 2800000
3762591 2900000
3880464 3000000
3998185 3100000
4115595 3200000
4233245 3300000
4351303 3400000
4469163 3500000
4587888 3600000
4706218 3700000
4824933 3800000
4943667 3900000
5060849 4000000
5177508 4100000
5293963 4200000
5410966 4300000
5528091 4400000
5645439 4500000
5763242 4600000
5881488 4700000
5999614 4800000
6117517 4900000
6235833 5000000
6354061 5100000
6471337 5200000
6588210 5300000
6706344 5400000
6823931 5500000
6941568 5600000
7057911 5700000
7177599 5800000
7297283 5900000
7418104 6000000
7550722 6100000
7669031 6200000
7794919 6300000
7913560 6400000
8030219 6500000
8149562 6600000
8296467 6700000
8420712 6800000
8541048 6900000
8664986 7000000
8782666 7100000
8900096 7200000
9016639 7300000
9133939 7400000
9251582 7500000
9369340 7600000
9486973 7700000
9604278 7800000
9721758 7900000
9839499 8000000
9957008 8100000
10074384 8200000
10197777 8300000
10321077 8400000
10443932 8500000
10573562 8600000
10707175 8700000
10835438 8800000
10965331 8900000
11094487 9000000
11225711 9100000
11365211 9200000
11508475 9300000
11642664 9400000
11776887 9500000
11917698 9600000
12066382 9700000
12208789 9800000
12350438 9900000
12498509 10000000
12643753 10100000
12771959 10200000
12901022 10300000
13028903 10400000
13154925 10500000
13282781 10600000
13414968 10700000
13551774 10800000
13691229 10900000
13815595 11000000
13948788 11100000
14084985 11200000
14209676 11300000
14333022 11400000
14456558 11500000
14581681 11600000
14710597 11700000
14834662 11800000
14959123 11900000
15081567 12000000
15206132 12100000
15331036 12200000
15456808 12300000
15582327 12400000
15706869 12500000
15830655 12600000
15952148 12700000
16072336 12800000
16193844 12900000
16314493 13000000
16436319 13100000
16559595 13200000
16699952 13300000
16823679 13400000
16943382 13500000
17061758 13600000
17182962 13700000
17304261 13800000
17424575 13900000
17545809 14000000
17667509 14100000
17789074 14200000
17910659 14300000
18032785 14400000
18157097 14500000
18281897 14600000
18404245 14700000
18526979 14800000
18649726 14900000
18772911 15000000
18894630 15100000
19017188 15200000
19138777 15300000
19261560 15400000
19386236 15500000
19510692 15600000
19635280 15700000
19757988 15800000
19880736 15900000
20003533 16000000
20130042 16100000
20256089 16200000
20380267 16300000
20510083 16400000
20634835 16500000
20758590 16600000
20881302 16700000
21004181 16800000
21127633 16900000
21250576 17000000
21376331 17100000
21501957 17200000
21626244 17300000
21752165 17400000
21876343 17500000
22000184 17600000
22126211 17700000
22249592 17800000
22373929 17900000
22498661 18000000
22622615 18100000
22748970 18200000
22875771 18300000
23005122 18400000
23129545 18500000
23256565 18600000
23383812 18700000
23512346 18800000
23638718 18900000
23763652 19000000
23890989 19100000
24027429 19200000
24156072 19300000
24290116 19400000
24422630 19500000
24560783 19600000
24694445 19700000
24826118 19800000
24960737 19900000
25105411 20000000
25242446 20100000
25372507 20200000
25504742 20300000
25629575 20400000
25758538 20500000
25889958 20600000
26019877 20700000
26153524 20800000
26288862 20900000
26424846 21000000
26557780 21100000
26694507 21200000
26832365 21300000
26971385 21400000
27108272 21500000
27258740 21600000
27385351 21700000
27519342 21800000
27653686 21900000
27785647 22000000
27912971 22100000
28050586 22200000
28200997 22300000
28345508 22400000
28487074 22500000
28610890 22600000
28733017 22700000
28856923 22800000
28981631 22900000
29121138 23000000
29258645 23100000
29390419 23200000
29516270 23300000
29647713 23400000
29778023 23500000
29903148 23600000
30033038 23700000
30165073 23800000
30297184 23900000
30420086 24000000
30543977 24100000
30667304 24200000
30793529 24300000
30919844 24400000
31050694 24500000
31181398 24600000
31308900 24700000
31437915 24800000
31562515 24900000
31684500 25000000
31808273 25100000
31935998 25200000
32057757 25300000
32190652 25400000
32317111 25500000
32444194 25600000
32569626 25700000
32693077 25800000
32816732 25900000
32936655 26000000
33061327 26100000
33185277 26200000
33308889 26300000
33432879 26400000
33558455 26500000
33687709 26600000
33814937 26700000
33939791 26800000
34063053 26900000
34185826 27000000
34307915 27100000
34435084 27200000
34560115 27300000
34687752 27400000
34814439 27500000
34946494 27600000
35080299 27700000
35203683 27800000
35331657 27900000
35464827 28000000
35589016 28100000
35713385 28200000
35837231 28300000
35960081 28400000
36087799 28500000
36218058 28600000
36349166 28700000
36484515 28800000
36612587 28900000
36744295 29000000
36869747 29100000
36996995 29200000
37122127 29300000
37244095 29400000
37374081 29500000
37506720 29600000
37639267 29700000
37772242 29800000
37906914 29900000
38036569 30000000
38160381 30100000
38286463 30200000
38415856 30300000
38550561 30400000
38688899 30500000
38823747 30600000
38955714 30700000
39089935 30800000
39229004 30900000
39366633 31000000
39499768 31100000
39634504 31200000
39764462 31300000
39896455 31400000
40027790 31500000
40160594 31600000
40295478 31700000
40429429 31800000
40563153 31900000
40690791 32000000
40822318 32100000
40954345 32200000
41099875 32300000
41229376 32400000
41360398 32500000
41489953 32600000
41622336 32700000
41787269 32800000
41917773 32900000
42046738 33000000
42179185 33100000
42309865 33200000
42441355 33300000
42571760 33400000
42701405 33500000
42832430 33600000
42964433 33700000
43099361 33800000
43232116 33900000
43365581 34000000
43497006 34100000
43632350 34200000
43763796 34300000
43895727 34400000
44032536 34500000
44168083 34600000
44349983 34700000
44497838 34800000
44643310 34900000
44776564 35000000
44908541 35100000
45041157 35200000
45176004 35300000
45311776 35400000
45445587 35500000
45581311 35600000
45711925 35700000
45843831 35800000
45975802 35900000
46110447 36000000
46244288 36100000
46375945 36200000
46508669 36300000
46639892 36400000
46772437 36500000
46907598 36600000
47039176 36700000
47180334 36800000
47311776 36900000
47443527 37000000
47575272 37100000
47707484 37200000
47838910 37300000
47969960 37400000
48128794 37500000
48286378 37600000
48442876 37700000
48606566 37800000
48762752 37900000
48924638 38000000
49083834 38100000
49243964 38200000
49401110 38300000
49560250 38400000
49715973 38500000
49876088 38600000
50034916 38700000
50194365 38800000
50349261 38900000
50508985 39000000
50665289 39100000
50820260 39200000
50976004 39300000
51138816 39400000
51294228 39500000
51451341 39600000
51607192 39700000
51767065 39800000
51925932 39900000
52082133 40000000
52238043 40100000
52391264 40200000
52545173 40300000
52698679 40400000
52856656 40500000
53011315 40600000
53170236 40700000
53329673 40800000
53487956 40900000
53648255 41000000
53804036 41100000
53961381 41200000
54117940 41300000
54275474 41400000
54434055 41500000
54592931 41600000
54753425 41700000
54910830 41800000
55069269 41900000
55226929 42000000
55385055 42100000
55545311 42200000
55704205 42300000
55862504 42400000
56022212 42500000
56181599 42600000
56339442 42700000
56500361 42800000
56658477 42900000
56816523 43000000
56974305 43100000
57135092 43200000
57295289 43300000
57454068 43400000
57613384 43500000
57770111 43600000
57926848 43700000
58079848 43800000
58237378 43900000
58393195 44000000
58549765 44100000
58704678 44200000
58859960 44300000
59016387 44400000
59173287 44500000
59330603 44600000
59486400 44700000
59641771 44800000
59796270 44900000
59951836 45000000
60093364 45100000
60245135 45200000
60398298 45300000
60548615 45400000
60706245 45500000
60862650 45600000
61019549 45700000
61176405 45800000
61332603 45900000
61492220 46000000
61648212 46100000
61807906 46200000
61967327 46300000
62124851 46400000
62283175 46500000
62439720 46600000
62598615 46700000
62757859 46800000
62916795 46900000
63074193 47000000
63226075 47100000
63385303 47200000
63542167 47300000
63701782 47400000
63863127 47500000
64020643 47600000
64178324 47700000
64334882 47800000
64489113 47900000
64641640 48000000
64797071 48100000
64949985 48200000
65100227 48300000
65251539 48400000
65402351 48500000
65558056 48600000
65709728 48700000
65858671 48800000
66000539 48900000
66145620 49000000
66290714 49100000
66437036 49200000
66584007 49300000
66730023 49400000
66875595 49500000
67023591 49600000
67163800 49700000
67303313 49800000
67439101 49900000
67572816 50000000
67712020 50100000
67850646 50200000
67985983 50300000
68121363 50400000
68259856 50500000
68384452 50600000
68509620 50700000
68642557 50800000
68776676 50900000
68908266 51000000
69037857 51100000
69169832 51200000
69303077 51300000
69437673 51400000
69561969 51500000
69696064 51600000
69846573 51700000
69999565 51800000
70153673 51900000
70299735 52000000
70439610 52100000
70581231 52200000
70726310 52300000
70878293 52400000
71030776 52500000
71176113 52600000
71318057 52700000
71461305 52800000
71612155 52900000
71759106 53000000
71900917 53100000
72048629 53200000
72197285 53300000
72352598 53400000
72483246 53500000
72617689 53600000
72751799 53700000
72878322 53800000
73006918 53900000
73132121 54000000
73261286 54100000
73387473 54200000
73515242 54300000
73646671 54400000
73780653 54500000
73915042 54600000
74053682 54700000
74190676 54800000
74332853 54900000
74486325 55000000
74636427 55100000
74787139 55200000
74933557 55300000
75082435 55400000
75214571 55500000
75336148 55600000
75475509 55700000
75613485 55800000
75749928 55900000
75896620 56000000
76046221 56100000
76189468 56200000
76332479 56300000
76479504 56400000
76601169 56500000
76743305 56600000
76888062 56700000
77031597 56800000
77179827 56900000
77310790 57000000
77452219 57100000
77599328 57200000
77747269 57300000
77896332 57400000
78036975 57500000
78186433 57600000
78328628 57700000
78478641 57800000
78624194 57900000
78759596 58000000
78886917 58100000
79012313 58200000
79153819 58300000
79273389 58400000
79421244 58500000
79568936 58600000
79718812 58700000
79865206 58800000
79997680 58900000
80131833 59000000
80249917 59100000
80363149 59200000
80481766 59300000
80601685 59400000
80730312 59500000
80869617 59600000
81034394 59700000
81196753 59800000
81355088 59900000
81507316 60000000
81658686 60100000
81805477 60200000
81953098 60300000
82078058 60400000
82213353 60500000
82355011 60600000
82512207 60700000
82666293 60800000
82799635 60900000
82936721 61000000
83095224 61100000
83254139 61200000
83386675 61300000
83540545 61400000
83700235 61500000
83843693 61600000
83996763 61700000
84151347 61800000
84309334 61900000
84464672 62000000
84622041 62100000
84780628 62200000
84934523 62300000
85087882 62400000
85245463 62500000
85401243 62600000
85556393 62700000
85716273 62800000
85877621 62900000
86033138 63000000
86188723 63100000
86343832 63200000
86497314 63300000
86649555 63400000
86802627 63500000
86957593 63600000
87111052 63700000
87264301 63800000
87423104 63900000
87582776 64000000
87740595 64100000
87896248 64200000
88054012 64300000
88210274 64400000
88362414 64500000
88516916 64600000
88671491 64700000
88826633 64800000
88982135 64900000
89137963 65000000
89290698 65100000
89440870 65200000
89585045 65300000
89730623 65400000
89877192 65500000
90024452 65600000
90172819 65700000
90320429 65800000
90469138 65900000
90618307 66000000
90772067 66100000
90923323 66200000
91074566 66300000
91226213 66400000
91376734 66500000
91523070 66600000
91667714 66700000
91816694 66800000
91966319 66900000
92112658 67000000
92261156 67100000
92406702 67200000
92548875 67300000
92692663 67400000
92838534 67500000
92986159 67600000
93135917 67700000
93284293 67800000
93430657 67900000
93575887 68000000
93723002 68100000
93875149 68200000
94024225 68300000
94170238 68400000
94316714 68500000
94463933 68600000
94610550 68700000
94757260 68800000
94903813 68900000
95051143 69000000
95199079 69100000
95349118 69200000
95496655 69300000
95645648 69400000
95791736 69500000
95940172 69600000
96090286 69700000
96239818 69800000
96387768 69900000
96525293 70000000
96668854 70100000
96812351 70200000
96959711 70300000
97101762 70400000
97236503 70500000
97371130 70600000
97501207 70700000
97648182 70800000
97796635 70900000
97944914 71000000
98093401 71100000
98213542 71200000
98328084 71300000
98444902 71400000
98558261 71500000
98693389 71600000
98849066 71700000
98998978 71800000
99145310 71900000
99290051 72000000
99440368 72100000
99585943 72200000
99728533 72300000
99873642 72400000
100019347 72500000
100173013 72600000
100326974 72700000
100477412 72800000
100627637 72900000
100777090 73000000
100920501 73100000
101061099 73200000
101209758 73300000
101355166 73400000
101497960 73500000
101641431 73600000
101787572 73700000
101927525 73800000
102071485 73900000
102219288 74000000
102362842 74100000
102505236 74200000
102652176 74300000
102798873 74400000
102941049 74500000
103088539 74600000
103242637 74700000
103389124 74800000
103544579 74900000
103719807 75000000
103864802 75100000
104007268 75200000
104149641 75300000
104292698 75400000
104434090 75500000
104577764 75600000
104721136 75700000
104863449 75800000

In [ ]:

    

np.save('D:\\vk_1015\\memcount',memcount)


    

In [ ]:

    

fillval = pack('i', 0)
with open(invfname, 'wb') as f:
    for uid, cnt in enumerate(memcount):
        if cnt > 0:
            f.write(pack('i', -uid))
            for i in range(int(cnt)):
                f.write(fillval)


    

In [ ]:

    

findex = np.zeros(n_nodes, dtype=np.int)
pos = 1
for uid, cnt in enumerate(memcount):
    findex[uid] = pos
    if cnt > 0:
        pos += cnt + 1


    

In [ ]:

    

print findex[:10]


    

In [ ]:

    

buffer_size = 1024
last_node = 1
cnt = 0
with open(invfname, 'r+') as outf:
    with open(fname, 'rb') as f:
        while True:
            bytes = f.read(4*buffer_size)
            if len(bytes) == 0:
                break
            buffer_values = unpack('i'*(len(bytes)/4), bytes)
            for idx, value in enumerate(buffer_values):
                if value <= 0:
                    last_node = -value
                    cnt += 1
                    if cnt%100000==0:
                        print last_node, cnt
                else:
                    if value == 0:
                        print 'writing', last_node, 'to', findex[value] * 4, 'value', value
                    if findex[value] * 4 == 0:
                        print 'writing', last_node, 'to', findex[value] * 4, 'value', value
                    outf.seek(findex[value] * 4, 0)
                    findex[value] += 1
                    outf.write(pack('i', last_node))
            del(bytes)


    

In [ ]:

    

buffer_size = 1024
last_node = 1
cnt = 0
with open(invfname, 'rb') as f:
    while True:
        bytes = f.read(4*buffer_size)
        if len(bytes) == 0:
            break
        buffer_values = unpack('i'*(len(bytes)/4), bytes)
        print buffer_values
        break


    

In [ ]:

    




    

In [ ]:

    




    

In [ ]: