In [1]:
!pip show systemml


You are using pip version 7.1.2, however version 9.0.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.

In [ ]:
!echo 'y' | pip uninstall systemml

In [2]:
!pip install --upgrade /Users/asurve/git/systemml_dl/systemml/target/systemml-1.0.0-SNAPSHOT-python.tgz


Processing /Users/asurve/git/systemml_dl/systemml/target/systemml-1.0.0-SNAPSHOT-python.tgz
Requirement already up-to-date: numpy>=1.8.2 in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from systemml==1.0.0)
Requirement already up-to-date: scipy>=0.15.1 in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from systemml==1.0.0)
Requirement already up-to-date: pandas in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from systemml==1.0.0)
Requirement already up-to-date: Pillow>=2.0.0 in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from systemml==1.0.0)
Requirement already up-to-date: pytz>=2011k in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from pandas->systemml==1.0.0)
Requirement already up-to-date: python-dateutil>=2 in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from pandas->systemml==1.0.0)
Requirement already up-to-date: olefile in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from Pillow>=2.0.0->systemml==1.0.0)
Requirement already up-to-date: six>=1.5 in /Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages (from python-dateutil>=2->pandas->systemml==1.0.0)
Installing collected packages: systemml
  Running setup.py install for systemml
Successfully installed systemml-1.0.0
You are using pip version 7.1.2, however version 9.0.1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.

In [3]:
from systemml import MLContext
ml = MLContext(sc)
print ("SystemML Built-Time:"+ ml.buildTime())
print(ml.info())


SystemML Built-Time:2017-06-27 04:30:02 UTC
Archiver-Version: Plexus Archiver
Artifact-Id: systemml
Build-Jdk: 1.8.0_121
Build-Time: 2017-06-27 04:30:02 UTC
Built-By: asurve
Created-By: Apache Maven 3.3.9
Group-Id: org.apache.systemml
Main-Class: org.apache.sysml.api.DMLScript
Manifest-Version: 1.0
Minimum-Recommended-Spark-Version: 2.1.0
Version: 1.0.0-SNAPSHOT


In [3]:
from systemml.mllearn import Caffe2DML
from pyspark.sql import SQLContext
import numpy as np
import urllib, os, scipy.ndimage
from PIL import Image
import systemml as sml

In [10]:
# ImageNet specific parameters
img_shape = (3, 224, 224)
num_classes = 1000

In [11]:
# Downloads a jpg image, resizes it to 224 and return as numpy array in N X CHW format
url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/58/MountainLion.jpg/312px-MountainLion.jpg'
outFile = 'test.jpg'
# urllib.urlretrieve(url, outFile)
input_image = sml.convertImageToNumPyArr(Image.open(outFile), img_shape=img_shape)

In [12]:
# Download the VGG network
# urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/vgg/ilsvrc12/VGG_ILSVRC_19_layers_solver.proto', 'VGG_ILSVRC_19_layers_solver.proto')
# urllib.urlretrieve('https://raw.githubusercontent.com/niketanpansare/model_zoo/master/caffe/vision/vgg/ilsvrc12/VGG_ILSVRC_19_layers_network.proto', 'VGG_ILSVRC_19_layers_network.proto')
  1. Download deployment file (.prototxt)
  2. Download caffemodel (.caffemodel)

In [13]:
# urllib.urlretrieve('http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_19_layers.caffemodel', 'VGG_ILSVRC_19_layers.caffemodel')

In [ ]:
home_dir = os.path.expanduser('~')
vgg_pretrained_weight_dir = os.path.join(home_dir, 'model_zoo', 'caffe', 'vision', 'vgg', 'ilsvrc12', 'VGG_ILSVRC_19_pretrained_weights')
print ('Pretrained dir: ' + vgg_pretrained_weight_dir)
# os.makedirs(vgg_pretrained_weight_dir)
sml.convert_caffemodel(SparkContext.getOrCreate(), '/Users/asurve/git/systemml_dl/systemml/samples/jupyter-notebooks/VGG_ILSVRC_19_layers_deploy.prototxt', '/Users/asurve/git/systemml_dl/systemml/samples/jupyter-notebooks/VGG_ILSVRC_19_layers.caffemodel', vgg_pretrained_weight_dir, format="binary", is_caffe_installed=False)
# sml.convert_caffemodel(sc, 'VGG_ILSVRC_19_layers_network.proto', 'VGG_ILSVRC_19_layers.caffemodel', vgg_pretrained_weight_dir, format="binary", is_caffe_installed=False)

In [ ]:
# Load the pretrained model and predict the downloaded image
# home_dir = os.path.expanduser('~')
# vgg_pretrained_weight_dir = os.path.join(home_dir, 'model_zoo', 'caffe', 'vision', 'vgg', 'ilsvrc12', 'VGG_ILSVRC_19_pretrained_weights')
vgg = Caffe2DML(sc, solver='VGG_ILSVRC_19_layers_solver.proto',  input_shape=img_shape)
vgg.set(debug=True)
vgg.load(vgg_pretrained_weight_dir)
vgg.predict(input_image)

In [14]:
# Load the pretrained model and predict the downloaded image
home_dir = os.path.expanduser('~')
alexnet_pretrained_weight_dir = os.path.join(home_dir, 'Documents/Docs/SystemML/models/Caffe', 'weight_dir/bvlc_alexnet')

sparkSession = SparkSession.builder \
     .master("local") \
     .appName("Alexnet Model Test") \
     .getOrCreate()

alexnet = Caffe2DML(sparkSession, solver='/Users/asurve/Documents/Docs/SystemML/models/Caffe/prototxt/bvlc_alexnet/solver.prototxt', netFilePath='/Users/asurve/Documents/Docs/SystemML/models/Caffe/prototxt/bvlc_alexnet/train_val.prototxt', input_shape=img_shape)
alexnet.set(debug=True)
alexnet.load(alexnet_pretrained_weight_dir)
# alexnet.predict(input_image)

In [15]:
alexnet.predict(input_image)


<class 'list'>
Out[15]:
array(['toilet tissue, toilet paper, bathroom tissue'],
      dtype='<U44')

In [5]:
# Step 1: Download the VGG network
import urllib
urllib.request.urlretrieve('https://raw.githubusercontent.com/niketanpansare/systemml/deconv/scripts/nn/examples/caffe2dml/models/imagenet/vgg19/VGG_ILSVRC_19_layers_deploy.proto', 'VGG_ILSVRC_19_layers_deploy.proto')
urllib.request.urlretrieve('https://raw.githubusercontent.com/niketanpansare/systemml/deconv/scripts/nn/examples/caffe2dml/models/imagenet/vgg19/VGG_ILSVRC_19_layers_network.proto', 'VGG_ILSVRC_19_layers_network.proto')
urllib.request.urlretrieve('https://raw.githubusercontent.com/niketanpansare/systemml/deconv/scripts/nn/examples/caffe2dml/models/imagenet/vgg19/VGG_ILSVRC_19_layers_solver.proto', 'VGG_ILSVRC_19_layers_solver.proto')
#urllib.request.urlretrieve('http://www.robots.ox.ac.uk/~vgg/software/very_deep/caffe/VGG_ILSVRC_19_layers.caffemodel', 'VGG_ILSVRC_19_layers.caffemodel')


Out[5]:
('VGG_ILSVRC_19_layers_solver.proto', <http.client.HTTPMessage at 0x107862f98>)

In [6]:
# Step 2: Convert the caffemodel to trained_vgg_weights directory
import systemml as sml
trained_vgg_weights = 'trained_vgg_weights'
sml.convert_caffemodel(sc, 'VGG_ILSVRC_19_layers_deploy.proto', 'VGG_ILSVRC_19_layers.caffemodel', trained_vgg_weights)
urllib.request.urlretrieve('https://raw.githubusercontent.com/niketanpansare/systemml/deconv/scripts/nn/examples/caffe2dml/models/imagenet/labels.txt', os.path.join(trained_vgg_weights, 'labels.txt'))


---------------------------------------------------------------------------
Py4JJavaError                             Traceback (most recent call last)
<ipython-input-6-92e2048e4b4a> in <module>()
      2 import systemml as sml
      3 trained_vgg_weights = 'trained_vgg_weights'
----> 4 sml.convert_caffemodel(sc, 'VGG_ILSVRC_19_layers_deploy.proto', 'VGG_ILSVRC_19_layers.caffemodel', trained_vgg_weights)
      5 urllib.request.urlretrieve('https://raw.githubusercontent.com/niketanpansare/systemml/deconv/scripts/nn/examples/caffe2dml/models/imagenet/labels.txt', os.path.join(trained_vgg_weights, 'labels.txt'))

/Users/asurve/.pyenv/versions/3.5.0/lib/python3.5/site-packages/systemml/converters.py in convert_caffemodel(sc, deploy_file, caffemodel_file, output_dir, format, is_caffe_installed)
    103         createJavaObject(sc, 'dummy')
    104         utilObj = sc._jvm.org.apache.sysml.api.dl.Utils()
--> 105         utilObj.saveCaffeModelFile(sc._jsc, deploy_file, caffemodel_file, output_dir, format)
    106 
    107 

/Users/asurve/Documents/Docs/SystemML/Spark/downloads/spark-2.0.2-bin-hadoop2.7/python/lib/py4j-0.10.3-src.zip/py4j/java_gateway.py in __call__(self, *args)
   1131         answer = self.gateway_client.send_command(command)
   1132         return_value = get_return_value(
-> 1133             answer, self.gateway_client, self.target_id, self.name)
   1134 
   1135         for temp_arg in temp_args:

/Users/asurve/Documents/Docs/SystemML/Spark/downloads/spark-2.0.2-bin-hadoop2.7/python/pyspark/sql/utils.py in deco(*a, **kw)
     61     def deco(*a, **kw):
     62         try:
---> 63             return f(*a, **kw)
     64         except py4j.protocol.Py4JJavaError as e:
     65             s = e.java_exception.toString()

/Users/asurve/Documents/Docs/SystemML/Spark/downloads/spark-2.0.2-bin-hadoop2.7/python/lib/py4j-0.10.3-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
    317                 raise Py4JJavaError(
    318                     "An error occurred while calling {0}{1}{2}.\n".
--> 319                     format(target_id, ".", name), value)
    320             else:
    321                 raise Py4JError(

Py4JJavaError: An error occurred while calling o54.saveCaffeModelFile.
: java.lang.OutOfMemoryError: Java heap space
	at org.apache.sysml.runtime.matrix.data.MatrixBlock.allocateDenseBlock(MatrixBlock.java:362)
	at org.apache.sysml.runtime.matrix.data.MatrixBlock.allocateDenseBlock(MatrixBlock.java:336)
	at org.apache.sysml.api.dl.Utils$.allocateMatrixBlock(Utils.scala:140)
	at org.apache.sysml.api.dl.Utils$$anonfun$readCaffeNet$1.apply(Utils.scala:218)
	at org.apache.sysml.api.dl.Utils$$anonfun$readCaffeNet$1.apply(Utils.scala:189)
	at scala.collection.Iterator$class.foreach(Iterator.scala:893)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
	at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at org.apache.sysml.api.dl.Utils$.readCaffeNet(Utils.scala:189)
	at org.apache.sysml.api.dl.Utils$.saveCaffeModelFile(Utils.scala:162)
	at org.apache.sysml.api.dl.Utils$.saveCaffeModelFile(Utils.scala:157)
	at org.apache.sysml.api.dl.Utils.saveCaffeModelFile(Utils.scala:298)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:280)
	at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
	at py4j.commands.CallCommand.execute(CallCommand.java:79)
	at py4j.GatewayConnection.run(GatewayConnection.java:214)
	at java.lang.Thread.run(Thread.java:745)

In [ ]: