notebook.community

Edit and run 





In [1]:



    


!pip3 install tqdm requests dill



    


















    






Requirement already satisfied (use --upgrade to upgrade): tqdm in /usr/local/lib/python3.5/dist-packages
Requirement already satisfied (use --upgrade to upgrade): requests in /usr/local/lib/python3.5/dist-packages
Requirement already satisfied (use --upgrade to upgrade): dill in /usr/local/lib/python3.5/dist-packages
Requirement already satisfied (use --upgrade to upgrade): idna<2.8,>=2.5 in /usr/local/lib/python3.5/dist-packages (from requests)
Requirement already satisfied (use --upgrade to upgrade): certifi>=2017.4.17 in /usr/local/lib/python3.5/dist-packages (from requests)
Requirement already satisfied (use --upgrade to upgrade): chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.5/dist-packages (from requests)
Requirement already satisfied (use --upgrade to upgrade): urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.5/dist-packages (from requests)
You are using pip version 8.1.1, however version 18.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.

















In [2]:



    


import requests
from tqdm import tqdm
import os

def download_from_url(url, dst):
    file_size = int(requests.head(url).headers["Content-Length"])
    if os.path.exists(dst):
        first_byte = os.path.getsize(dst)
    else:
        first_byte = 0
    if first_byte >= file_size:
        return file_size
    header = {"Range": "bytes=%s-%s" % (first_byte, file_size)}
    pbar = tqdm(
        total=file_size, initial=first_byte,
        unit='B', unit_scale=True, desc=url.split('/')[-1])
    req = requests.get(url, headers=header, stream=True)
    with(open(dst, 'ab')) as f:
        for chunk in req.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                pbar.update(1024)
    pbar.close()
    return file_size



    











In [3]:



    


download_from_url('https://raw.githubusercontent.com/sjwhitworth/golearn/master/examples/datasets/mnist_train.csv', 
                  'mnist_train.csv')



    


















    
Out[3]:








1115034

















In [4]:



    


from sparkflow.graph_utils import build_graph
from sparkflow.tensorflow_async import SparkAsyncDL
import tensorflow as tf
from pyspark.ml.feature import VectorAssembler, OneHotEncoder
from pyspark.ml.pipeline import Pipeline
from sparkflow.graph_utils import build_adam_config
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
    
def cnn_model():
    x = tf.placeholder(tf.float32, shape=[None, 784], name='x')
    y = tf.placeholder(tf.float32, shape=[None, 10], name='y')
    x = tf.reshape(x, shape=[-1, 28, 28, 1])
    conv1 = tf.layers.conv2d(x, 32, 5, activation=tf.nn.relu)
    conv1 = tf.layers.max_pooling2d(conv1, 2, 2)
    conv2 = tf.layers.conv2d(conv1, 64, 3, activation=tf.nn.relu)
    conv2 = tf.layers.max_pooling2d(conv2, 2, 2)
    fc1 = tf.contrib.layers.flatten(conv2)
    out = tf.layers.dense(fc1, 10)
    z = tf.argmax(out, 1, name='out')
    loss = tf.losses.softmax_cross_entropy(y, out)
    return loss



    











In [5]:



    


from pyspark.sql import SparkSession
from pyspark.sql.functions import rand
sparkSession = SparkSession.builder.appName("csv").getOrCreate()



    











In [6]:



    


df = sparkSession.read.csv('mnist_train.csv',header=True,inferSchema=True)



    











In [7]:



    


va = VectorAssembler(inputCols=df.columns[1:785], outputCol='features').transform(df)



    











In [8]:



    


va.select('label').show(1)



    


















    






+-----+
|label|
+-----+
|    1|
+-----+
only showing top 1 row


















In [9]:



    


encoded = OneHotEncoder(inputCol='label', outputCol='labels', dropLast=False).transform(va).select(['features', 'labels'])



    











In [10]:



    


mg = build_graph(cnn_model)
adam_config = build_adam_config(learning_rate=0.001, beta1=0.9, beta2=0.999)



    











In [11]:



    


spark_model = SparkAsyncDL(
    inputCol='features',
    tensorflowGraph=mg,
    tfInput='x:0',
    tfLabel='y:0',
    tfOutput='out:0',
    tfOptimizer='adam',
    miniBatchSize=300,
    miniStochasticIters=1,
    shufflePerIter=True,
    iters=50,
    predictionCol='predicted',
    labelCol='labels',
    partitions=3,
    verbose=1,
    optimizerOptions=adam_config
)



    











In [13]:



    


fitted_model = spark_model.fit(encoded)



    


















    






 * Serving Flask app "sparkflow.HogwildSparkModel" (lazy loading)
 * Environment: production
   WARNING: Do not use the development server in a production environment.
   Use a production WSGI server instead.
 * Debug mode: on










    






 * Running on http://0.0.0.0:5000/ (Press CTRL+C to quit)
172.23.0.2 - - [22/Nov/2018 06:21:01] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:03] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:03] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:04] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:04] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:05] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:05] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:06] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:06] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:06] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:07] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:07] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:08] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:08] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:08] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:09] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:09] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:10] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:10] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:11] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:11] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:12] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:12] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:13] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:13] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:14] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:14] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:15] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:15] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:16] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:16] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:16] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:17] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:17] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:17] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:18] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:19] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:19] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:19] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:20] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:20] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:21] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:21] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:22] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:22] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:23] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:23] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:23] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:23] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:24] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:24] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:25] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:25] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:25] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:26] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:26] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:26] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:27] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:27] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:27] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:27] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:28] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:28] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:28] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:28] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:29] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:29] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:29] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:30] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:30] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:31] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:31] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:31] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:31] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:32] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:32] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:33] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:33] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:33] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:34] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:34] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:34] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:34] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:35] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:35] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:35] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:36] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:36] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:36] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:37] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:37] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:37] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:37] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:38] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:38] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:38] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:39] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:39] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:39] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:40] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:40] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:40] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:41] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:41] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:41] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:42] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:42] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:43] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:43] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:43] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:43] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:44] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:44] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:44] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:44] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:45] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:45] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:45] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:46] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:46] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:46] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:47] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:47] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:47] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:48] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:48] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:48] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:49] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:49] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:49] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:50] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:50] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:50] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:50] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:51] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:51] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:52] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:52] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:52] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:53] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:53] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:53] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:53] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:54] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:54] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:54] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:54] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:55] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:55] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:56] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:56] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:56] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:56] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:57] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:57] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:57] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:57] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:58] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:58] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:59] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:59] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:21:59] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:00] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:00] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:00] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:01] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:01] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:01] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:02] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:02] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:03] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:03] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:03] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:04] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:04] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:04] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:05] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:05] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:06] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:06] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:07] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:07] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:07] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:08] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:08] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:08] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:09] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:09] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:10] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:10] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:10] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:11] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:11] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:11] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:12] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:12] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:13] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:13] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:14] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:14] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:14] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:15] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:15] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:16] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:16] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:16] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:17] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:17] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:18] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:18] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:18] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:19] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:19] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:20] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:20] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:21] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:21] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:21] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:22] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:22] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:22] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:23] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:23] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:24] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:24] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:24] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:25] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:25] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:25] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:26] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:26] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:27] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:27] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:28] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:28] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:28] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:28] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:29] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:29] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:30] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:30] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:31] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:31] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:31] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:32] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:32] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:32] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:33] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:33] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:34] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:34] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:35] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:35] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:35] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:36] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:36] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:36] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:37] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:37] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:38] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:38] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:39] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:39] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:39] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:40] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:40] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:40] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:41] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:41] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:42] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:42] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:43] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:43] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:43] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:44] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:44] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:44] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:45] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:45] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:46] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:46] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:47] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:47] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:48] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:48] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:48] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:48] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:49] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:49] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:50] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:50] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:51] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:51] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:51] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:52] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:52] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:52] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:53] "GET /parameters HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:53] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:54] "POST /update HTTP/1.1" 200 -
172.23.0.2 - - [22/Nov/2018 06:22:54] "GET /parameters HTTP/1.1" 200 -

















In [14]:



    


predictions = fitted_model.transform(encoded)



    











In [15]:



    


predictions.show(1)



    


















    






+--------------------+--------------+---------+
|            features|        labels|predicted|
+--------------------+--------------+---------+
|(784,[132,133,134...|(10,[1],[1.0])|       []|
+--------------------+--------------+---------+
only showing top 1 row


















In [ ]:



    


evaluator = MulticlassClassificationEvaluator(
    labelCol="labels", predictionCol="predicted", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test Error = %g" % (1.0 - accuracy))



    











In [ ]:

Content source: huseinzol05/Deep-Learning-Tensorflow

Similar notebooks:

notebook.community | gallery | about