Fuel data processing server

In [ ]:

    

import time

from fuel.datasets import IndexableDataset
from fuel.schemes import ShuffledScheme
from fuel.server import start_server
from fuel.streams import DataStream
from fuel.transformers import Transformer


class Bottleneck(Transformer):
    def __init__(self, data_stream, **kwargs):
        self.slowdown = kwargs.pop('slowdown', 0)
        super(Bottleneck, self).__init__(
            data_stream, data_stream.produces_examples, **kwargs)

    def get_data(self, request=None):
        if request is not None:
            raise ValueError
        time.sleep(self.slowdown)
        return next(self.child_epoch_iterator)


def create_data_stream(slowdown=0):
    dataset = IndexableDataset({'features': [[0] * 128] * 1000})
    iteration_scheme = ShuffledScheme(examples=1000, batch_size=100)
    data_stream = Bottleneck(
        data_stream=DataStream.default_stream(
            dataset=dataset, iteration_scheme=iteration_scheme),
        slowdown=slowdown)
    return data_stream


    

In [ ]:

    

start_server(create_data_stream(0.005))