In [1]:

    

from bcolz_array_iterator2 import BcolzArrayIterator2


    

In [2]:

    

from bcolz import carray


    

In [3]:

    

import numpy as np


    

In [4]:

    

x = np.arange(14); x


    

    
Out[4]:





array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13])

In [5]:

    

y = np.arange(14); y


    

    
Out[5]:





array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13])

In [16]:

    

x = carray(x, chunklen=3)
y = carray(y, chunklen=3)


    

In [17]:

    

b = BcolzArrayIterator2(x, y, shuffle=True, batch_size=3)


    

In [18]:

    

b.N


    

    
Out[18]:





14

In [19]:

    

nit = len(x)//b.batch_size+1; nit


    

    
Out[19]:





5

In [20]:

    

for j in range(10000):
    bx,by = list(zip(*[next(b) for i in range(nit)]))
    nx = np.concatenate(bx)
    ny = np.concatenate(by)
    assert(np.allclose(nx,ny))
    assert(len(np.unique(nx))==len(nx))


    

In [21]:

    

[next(b) for i in range(20)]


    

    
Out[21]:





[(array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([0, 1, 2]), array([0, 1, 2])),
 (array([12, 13]), array([12, 13])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([0, 1, 2]), array([0, 1, 2])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([12, 13]), array([12, 13])),
 (array([12, 13]), array([12, 13])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([0, 1, 2]), array([0, 1, 2])),
 (array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([3, 4, 5]), array([3, 4, 5])),
 (array([6, 7, 8]), array([6, 7, 8])),
 (array([ 9, 10, 11]), array([ 9, 10, 11])),
 (array([12, 13]), array([12, 13])),
 (array([0, 1, 2]), array([0, 1, 2]))]

In [ ]: