In [1]:
import logging
import importlib
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
importlib.reload(logging)
logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%I:%M:%S')
In [2]:
import karps as ks
import karps.functions as f
from karps.display import show_phase
In [3]:
df = ks.dataframe([1.0, 2.0], name="my_input")
df
Out[3]:
/my_input@org.spark.DistributedLiteral:double
In [4]:
s = f.sum(df)
c = f.as_double(f.count(df))
In [5]:
m = s / c
m
Out[5]:
/divide3!org.spark.LocalStructuredTransform:double
In [6]:
s = ks.session("test3", address="ec2-52-35-61-191.us-west-2.compute.amazonaws.com", port=6006)
comp = s.compute(m)
04:14:15 DEBUG:compute graph: nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE }}"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\013\n\t\n\005count\022\000"
content_debug: "agg_op { op { function_name: \"count\" inputs { } }}"
}
parents {
path: "my_input"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\006\"\000R\002_0"
content_debug: "col_op { function { function_name: \"cast_double\" inputs { broadcast { } field_name: \"_0\" } }}"
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\t\n\007\n\003sum\022\000"
content_debug: "agg_op { op { function_name: \"sum\" inputs { } }}"
}
parents {
path: "my_input"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\034\022\032\n\006divide\022\006\"\000R\002_0\022\010\"\002\010\001R\002_1"
content_debug: "col_op { function { function_name: \"divide\" inputs { broadcast { } field_name: \"_0\" } inputs { broadcast { observable_index: 1 } field_name: \"_1\" } }}"
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
basic_type: DOUBLE
}
}
In [7]:
show_phase(comp, "initial")
04:14:15 WARNING:Could not find compiler step initial. Available steps are [] and []
In [8]:
show_phase(comp, "REMOVE_OBSERVABLE_BROADCASTS")
04:14:15 WARNING:Could not find compiler step REMOVE_OBSERVABLE_BROADCASTS. Available steps are [] and []
In [9]:
show_phase(comp, "MERGE_AGGREGATIONS")
04:14:15 WARNING:Could not find compiler step MERGE_AGGREGATIONS. Available steps are [] and []
In [17]:
show_phase(comp, "final")
In [11]:
comp.values()
04:14:15 DEBUG:Calling _progress
04:14:16 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
start_graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE }}"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\013\n\t\n\005count\022\000"
content_debug: "agg_op { op { function_name: \"count\" inputs { } }}"
}
parents {
path: "my_input"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\006\"\000R\002_0"
content_debug: "col_op { function { function_name: \"cast_double\" inputs { broadcast { } field_name: \"_0\" } }}"
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\t\n\007\n\003sum\022\000"
content_debug: "agg_op { op { function_name: \"sum\" inputs { } }}"
}
parents {
path: "my_input"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\034\022\032\n\006divide\022\006\"\000R\002_0\022\010\"\002\010\001R\002_1"
content_debug: "col_op { function { function_name: \"divide\" inputs { broadcast { } field_name: \"_0\" } inputs { broadcast { observable_index: 1 } field_name: \"_1\" } }}"
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
basic_type: DOUBLE
}
}
}
pinned_graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
compilation_result {
compilation_graph {
phase_name: "INITIAL"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\017\n\r\n\005count\022\000\032\002\010\001"
content_debug: "agg_op {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n}"
content_base64: "Cg8KDQoFY291bnQSABoCCAE="
}
parents {
path: "my_input"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\"\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { broadcast { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAiIAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\r\n\013\n\003sum\022\000\032\002\010\002"
content_debug: "agg_op {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Cg0KCwoDc3VtEgAaAggC"
}
parents {
path: "my_input"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\030\022\026\n\006divide\022\002\"\000\022\004\"\002\010\001\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { broadcast { } }\n inputs { broadcast { observable_index: 1 } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChgSFgoGZGl2aWRlEgIiABIEIgIIARoCCAI="
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "count1"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "agg_op { op { function_name: .count. inputs { } expected_type { basic_type: INT } }}"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { broadcast { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "agg_op { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { broadcast { } } inputs { broadcast { observable_index: 1 } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "REMOVE_OBSERVABLE_BROADCASTS"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\017\n\r\n\005count\022\000\032\002\010\001"
content_debug: "agg_op {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n}"
content_base64: "Cg8KDQoFY291bnQSABoCCAE="
}
parents {
path: "my_input"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\r\n\013\n\003sum\022\000\032\002\010\002"
content_debug: "agg_op {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Cg0KCwoDc3VtEgAaAggC"
}
parents {
path: "my_input"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "count1"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "agg_op { op { function_name: .count. inputs { } expected_type { basic_type: INT } }}"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "agg_op { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "MERGE_PREAGG_AGGREGATIONS"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\017\n\r\n\005count\022\000\032\002\010\001"
content_debug: "agg_op {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n}"
content_base64: "Cg8KDQoFY291bnQSABoCCAE="
}
parents {
path: "my_input"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n\r\n\013\n\003sum\022\000\032\002\010\002"
content_debug: "agg_op {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Cg0KCwoDc3VtEgAaAggC"
}
parents {
path: "my_input"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "count1"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "agg_op { op { function_name: .count. inputs { } expected_type { basic_type: INT } }}"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "agg_op { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "MERGE_AGGREGATIONS"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "my_input_karps_merged_agg"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:int _2:double}"
}
}
attr {
key: "zextra"
value {
s: "agg_op { struct { fields { op { function_name: .count. inputs { } expected_type { basic_type: INT } } field_name: ._1. } fields { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } } field_name: ._2. } }}"
}
}
}
node {
name: "count1"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._1. } }"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._2. } }"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "DATA_SOURCE_INSERTION"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "my_input_karps_merged_agg"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:int _2:double}"
}
}
attr {
key: "zextra"
value {
s: "agg_op { struct { fields { op { function_name: .count. inputs { } expected_type { basic_type: INT } } field_name: ._1. } fields { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } } field_name: ._2. } }}"
}
}
}
node {
name: "count1"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._1. } }"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._2. } }"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "POINTER_SWAP_1"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "my_input_karps_merged_agg"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:int _2:double}"
}
}
attr {
key: "zextra"
value {
s: "agg_op { struct { fields { op { function_name: .count. inputs { } expected_type { basic_type: INT } } field_name: ._1. } fields { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } } field_name: ._2. } }}"
}
}
}
node {
name: "count1"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._1. } }"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._2. } }"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "FUNCTIONAL_FLATTENING"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "my_input_karps_merged_agg"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:int _2:double}"
}
}
attr {
key: "zextra"
value {
s: "agg_op { struct { fields { op { function_name: .count. inputs { } expected_type { basic_type: INT } } field_name: ._1. } fields { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } } field_name: ._2. } }}"
}
}
}
node {
name: "count1"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._1. } }"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._2. } }"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "AUTOCACHE_FULLFILL"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "my_input_karps_merged_agg"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:int _2:double}"
}
}
attr {
key: "zextra"
value {
s: "agg_op { struct { fields { op { function_name: .count. inputs { } expected_type { basic_type: INT } } field_name: ._1. } fields { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } } field_name: ._2. } }}"
}
}
}
node {
name: "count1"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._1. } }"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._2. } }"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
compilation_graph {
phase_name: "FINAL"
graph {
nodes {
locality: DISTRIBUTED
path {
path: "my_input"
}
op_name: "org.spark.DistributedLiteral"
op_extra {
content: "\n\030R\026\n\t)\000\000\000\000\000\000\360?\n\t)\000\000\000\000\000\000\000@\022\004\022\002\010\002"
content_debug: "cell {\n array_value {\n values { double_value: 1.0 } values { double_value: 2.0 }\n }\n}\ncell_type { array_type { basic_type: DOUBLE } }"
content_base64: "ChhSFgoJKQAAAAAAAPA/CgkpAAAAAAAAAEASBBICCAI="
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "my_input_karps_merged_agg"
}
op_name: "org.spark.StructuredReduce"
op_extra {
content: "\n*\022(\n\023\n\r\n\005count\022\000\032\002\010\001\032\002_1\n\021\n\013\n\003sum\022\000\032\002\010\002\032\002_2"
content_debug: "agg_op {\n struct {\n fields {\n op {\n function_name: \"count\" inputs { } expected_type { basic_type: INT }\n }\n field_name: \"_1\"\n }\n fields {\n op {\n function_name: \"sum\"\n inputs { }\n expected_type { basic_type: DOUBLE }\n }\n field_name: \"_2\"\n }\n }\n}"
content_base64: "CioSKAoTCg0KBWNvdW50EgAaAggBGgJfMQoRCgsKA3N1bRIAGgIIAhoCXzI="
}
parents {
path: "my_input"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "count1"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_1"
content_debug: "col_op { extraction { path: \"_1\" } }"
content_base64: "CgYaBAoCXzE="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: INT
}
}
nodes {
path {
path: "cast_double2"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\027\022\025\n\013cast_double\022\002\032\000\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"cast_double\"\n inputs { extraction { } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "ChcSFQoLY2FzdF9kb3VibGUSAhoAGgIIAg=="
}
parents {
path: "count1"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "sum0"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\006\032\004\n\002_2"
content_debug: "col_op { extraction { path: \"_2\" } }"
content_base64: "CgYaBAoCXzI="
}
parents {
path: "my_input_karps_merged_agg"
}
infered_type {
basic_type: DOUBLE
}
}
nodes {
path {
path: "divide3_karps_localpack"
}
op_name: "org.spark.LocalPack"
op_extra {
}
parents {
path: "sum0"
}
parents {
path: "cast_double2"
}
infered_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
nodes {
path {
path: "divide3"
}
op_name: "org.spark.LocalStructuredTransform"
op_extra {
content: "\n\036\022\034\n\006divide\022\006\032\004\n\002_1\022\006\032\004\n\002_2\032\002\010\002"
content_debug: "col_op {\n function {\n function_name: \"divide\"\n inputs { extraction { path: \"_1\" } }\n inputs { extraction { path: \"_2\" } }\n expected_type { basic_type: DOUBLE }\n }\n}"
content_base64: "Ch4SHAoGZGl2aWRlEgYaBAoCXzESBhoECgJfMhoCCAI="
}
parents {
path: "divide3_karps_localpack"
}
infered_type {
basic_type: DOUBLE
}
}
}
graph_def {
node {
name: "my_input"
op: "org.spark.DistributedLiteral"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Distributed"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "cell { array_value { values { double_value: 1.0 } values { double_value: 2.0 } }}cell_type { array_type { basic_type: DOUBLE } }"
}
}
}
node {
name: "my_input_karps_merged_agg"
op: "org.spark.StructuredReduce"
input: "my_input"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:int _2:double}"
}
}
attr {
key: "zextra"
value {
s: "agg_op { struct { fields { op { function_name: .count. inputs { } expected_type { basic_type: INT } } field_name: ._1. } fields { op { function_name: .sum. inputs { } expected_type { basic_type: DOUBLE } } field_name: ._2. } }}"
}
}
}
node {
name: "count1"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "int"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._1. } }"
}
}
}
node {
name: "cast_double2"
op: "org.spark.LocalStructuredTransform"
input: "count1"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .cast_double. inputs { extraction { } } expected_type { basic_type: DOUBLE } }}"
}
}
}
node {
name: "sum0"
op: "org.spark.LocalStructuredTransform"
input: "my_input_karps_merged_agg"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { extraction { path: ._2. } }"
}
}
}
node {
name: "divide3_karps_localpack"
op: "org.spark.LocalPack"
input: "sum0"
input: "cast_double2"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "{_1:double _2:double}"
}
}
attr {
key: "zextra"
value {
s: ""
}
}
}
node {
name: "divide3"
op: "org.spark.LocalStructuredTransform"
input: "divide3_karps_localpack"
device: "/spark:0"
attr {
key: "locality"
value {
s: "Local"
}
}
attr {
key: "type"
value {
s: "double"
}
}
attr {
key: "zextra"
value {
s: "col_op { function { function_name: .divide. inputs { extraction { path: ._1. } } inputs { extraction { path: ._2. } } expected_type { basic_type: DOUBLE } }}"
}
}
}
}
}
}
04:14:16 DEBUG:channel: received graph (discarding)
04:14:16 DEBUG:channel: received pinned graph (discarding)
04:14:16 DEBUG:channel: received compilation results
04:14:16 DEBUG:channel: received compilation steps
04:14:16 DEBUG:Calling _progress
04:14:16 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "my_input_karps_merged_agg"
}
status: SCHEDULED
}
results {
local_path {
path: "cast_double2"
}
status: SCHEDULED
}
results {
local_path {
path: "divide3_karps_localpack"
}
status: SCHEDULED
}
results {
local_path {
path: "my_input"
}
status: SCHEDULED
}
results {
local_path {
path: "sum0"
}
status: SCHEDULED
}
results {
local_path {
path: "divide3"
}
status: SCHEDULED
}
results {
local_path {
path: "count1"
}
status: SCHEDULED
}
}
04:14:16 DEBUG:channel: received result for /my_input_karps_merged_agg: local_path {
path: "my_input_karps_merged_agg"
}
status: SCHEDULED
04:14:16 DEBUG:channel: received result for /cast_double2: local_path {
path: "cast_double2"
}
status: SCHEDULED
04:14:16 DEBUG:channel: received result for /divide3_karps_localpack: local_path {
path: "divide3_karps_localpack"
}
status: SCHEDULED
04:14:16 DEBUG:channel: received result for /my_input: local_path {
path: "my_input"
}
status: SCHEDULED
04:14:16 DEBUG:channel: received result for /sum0: local_path {
path: "sum0"
}
status: SCHEDULED
04:14:16 DEBUG:channel: received result for /divide3: local_path {
path: "divide3"
}
status: SCHEDULED
04:14:16 DEBUG:channel: received result for /count1: local_path {
path: "count1"
}
status: SCHEDULED
04:14:16 DEBUG:Calling _progress
04:14:16 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "my_input"
}
status: SCHEDULED
spark_stats {
rdd_info {
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[0] at javaRDD at ExecutionItem.scala:56"
proto {
name: "my_input/ParallelCollectionRDD_0"
op: "ParallelCollectionRDD"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 1
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[1] at javaRDD at ExecutionItem.scala:56"
parents: 0
proto {
name: "my_input/MapPartitionsRDD_1"
op: "MapPartitionsRDD"
input: "my_input/ParallelCollectionRDD_0"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 2
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[2] at javaRDD at ExecutionItem.scala:56"
parents: 1
proto {
name: "my_input/MapPartitionsRDD_2"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_1"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 3
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[3] at javaRDD at ExecutionItem.scala:56"
parents: 2
proto {
name: "my_input/MapPartitionsRDD_3"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_2"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 4
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[4] at createDataFrame at ExecutionItem.scala:56"
parents: 3
proto {
name: "my_input/MapPartitionsRDD_4"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_3"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 5
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[5] at execute at ExecutionItem.scala:89"
parents: 4
proto {
name: "my_input/MapPartitionsRDD_5"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#93]"
proto {
name: "my_input/LogicalRDD_0"
op: "LogicalRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#93]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#93]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#93]"
proto {
name: "my_input/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#93]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#93]"
}
}
}
}
}
}
}
04:14:16 DEBUG:channel: received result for /my_input: local_path {
path: "my_input"
}
status: SCHEDULED
spark_stats {
rdd_info {
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[0] at javaRDD at ExecutionItem.scala:56"
proto {
name: "my_input/ParallelCollectionRDD_0"
op: "ParallelCollectionRDD"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 1
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[1] at javaRDD at ExecutionItem.scala:56"
parents: 0
proto {
name: "my_input/MapPartitionsRDD_1"
op: "MapPartitionsRDD"
input: "my_input/ParallelCollectionRDD_0"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 2
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[2] at javaRDD at ExecutionItem.scala:56"
parents: 1
proto {
name: "my_input/MapPartitionsRDD_2"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_1"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 3
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[3] at javaRDD at ExecutionItem.scala:56"
parents: 2
proto {
name: "my_input/MapPartitionsRDD_3"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_2"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 4
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[4] at createDataFrame at ExecutionItem.scala:56"
parents: 3
proto {
name: "my_input/MapPartitionsRDD_4"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_3"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 5
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[5] at execute at ExecutionItem.scala:89"
parents: 4
proto {
name: "my_input/MapPartitionsRDD_5"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#93]"
proto {
name: "my_input/LogicalRDD_0"
op: "LogicalRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#93]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#93]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#93]"
proto {
name: "my_input/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#93]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#93]"
}
}
}
}
}
04:14:16 DEBUG:Calling _progress
04:14:17 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "my_input"
}
status: SCHEDULED
spark_stats {
rdd_info {
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[0] at javaRDD at ExecutionItem.scala:56"
proto {
name: "my_input/ParallelCollectionRDD_0"
op: "ParallelCollectionRDD"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 1
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[1] at javaRDD at ExecutionItem.scala:56"
parents: 0
proto {
name: "my_input/MapPartitionsRDD_1"
op: "MapPartitionsRDD"
input: "my_input/ParallelCollectionRDD_0"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 2
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[2] at javaRDD at ExecutionItem.scala:56"
parents: 1
proto {
name: "my_input/MapPartitionsRDD_2"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_1"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 3
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[3] at javaRDD at ExecutionItem.scala:56"
parents: 2
proto {
name: "my_input/MapPartitionsRDD_3"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_2"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 4
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[4] at createDataFrame at ExecutionItem.scala:56"
parents: 3
proto {
name: "my_input/MapPartitionsRDD_4"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_3"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 5
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[5] at execute at ExecutionItem.scala:89"
parents: 4
proto {
name: "my_input/MapPartitionsRDD_5"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#93]"
proto {
name: "my_input/LogicalRDD_0"
op: "LogicalRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#93]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#93]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#93]"
proto {
name: "my_input/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#93]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#93]"
}
}
}
}
}
}
}
04:14:17 DEBUG:channel: received result for /my_input: local_path {
path: "my_input"
}
status: SCHEDULED
spark_stats {
rdd_info {
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[0] at javaRDD at ExecutionItem.scala:56"
proto {
name: "my_input/ParallelCollectionRDD_0"
op: "ParallelCollectionRDD"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 1
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[1] at javaRDD at ExecutionItem.scala:56"
parents: 0
proto {
name: "my_input/MapPartitionsRDD_1"
op: "MapPartitionsRDD"
input: "my_input/ParallelCollectionRDD_0"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 2
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[2] at javaRDD at ExecutionItem.scala:56"
parents: 1
proto {
name: "my_input/MapPartitionsRDD_2"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_1"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 3
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[3] at javaRDD at ExecutionItem.scala:56"
parents: 2
proto {
name: "my_input/MapPartitionsRDD_3"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_2"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 4
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[4] at createDataFrame at ExecutionItem.scala:56"
parents: 3
proto {
name: "my_input/MapPartitionsRDD_4"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_3"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 5
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[5] at execute at ExecutionItem.scala:89"
parents: 4
proto {
name: "my_input/MapPartitionsRDD_5"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#93]"
proto {
name: "my_input/LogicalRDD_0"
op: "LogicalRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#93]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#93]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#93]"
proto {
name: "my_input/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#93]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#93]"
}
}
}
}
}
04:14:17 DEBUG:Calling _progress
04:14:17 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "my_input_karps_merged_agg"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 6
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[6] at javaRDD at ExecutionItem.scala:56"
parents: 4
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_6"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
input: "^my_input/MapPartitionsRDD_5"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 7
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[7] at javaRDD at ExecutionItem.scala:56"
parents: 6
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_7"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_6"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 8
class_name: "ShuffledRowRDD"
repr: "ShuffledRowRDD[8] at javaRDD at ExecutionItem.scala:56"
parents: 7
proto {
name: "my_input_karps_merged_agg/ShuffledRowRDD_8"
op: "ShuffledRowRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_7"
attr {
key: "name"
value {
s: "ShuffledRowRDD"
}
}
}
}
rdd_info {
rdd_id: 9
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[9] at javaRDD at ExecutionItem.scala:56"
parents: 8
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_9"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/ShuffledRowRDD_8"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 10
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[10] at javaRDD at ExecutionItem.scala:56"
parents: 9
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_10"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_9"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 11
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[11] at javaRDD at ExecutionItem.scala:56"
parents: 10
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_11"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_10"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 12
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[12] at createDataFrame at ExecutionItem.scala:56"
parents: 11
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_12"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_11"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 13
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[13] at execute at ExecutionItem.scala:89"
parents: 12
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_13"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_12"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#128L, _2#129]"
proto {
name: "my_input_karps_merged_agg/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#128L,_2#129]"
proto {
name: "my_input_karps_merged_agg/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
}
}
}
}
}
04:14:17 DEBUG:channel: received result for /my_input_karps_merged_agg: local_path {
path: "my_input_karps_merged_agg"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 6
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[6] at javaRDD at ExecutionItem.scala:56"
parents: 4
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_6"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
input: "^my_input/MapPartitionsRDD_5"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 7
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[7] at javaRDD at ExecutionItem.scala:56"
parents: 6
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_7"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_6"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 8
class_name: "ShuffledRowRDD"
repr: "ShuffledRowRDD[8] at javaRDD at ExecutionItem.scala:56"
parents: 7
proto {
name: "my_input_karps_merged_agg/ShuffledRowRDD_8"
op: "ShuffledRowRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_7"
attr {
key: "name"
value {
s: "ShuffledRowRDD"
}
}
}
}
rdd_info {
rdd_id: 9
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[9] at javaRDD at ExecutionItem.scala:56"
parents: 8
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_9"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/ShuffledRowRDD_8"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 10
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[10] at javaRDD at ExecutionItem.scala:56"
parents: 9
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_10"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_9"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 11
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[11] at javaRDD at ExecutionItem.scala:56"
parents: 10
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_11"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_10"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 12
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[12] at createDataFrame at ExecutionItem.scala:56"
parents: 11
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_12"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_11"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 13
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[13] at execute at ExecutionItem.scala:89"
parents: 12
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_13"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_12"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#128L, _2#129]"
proto {
name: "my_input_karps_merged_agg/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#128L,_2#129]"
proto {
name: "my_input_karps_merged_agg/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
}
}
}
04:14:17 DEBUG:Calling _progress
04:14:25 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "my_input_karps_merged_agg"
}
status: FINISHED_SUCCESS
final_result {
cell {
struct_value {
values {
int_value: 2
}
values {
double_value: 3.0
}
}
}
cell_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
spark_stats {
rdd_info {
rdd_id: 6
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[6] at javaRDD at ExecutionItem.scala:56"
parents: 4
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_6"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
input: "^my_input/MapPartitionsRDD_5"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 7
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[7] at javaRDD at ExecutionItem.scala:56"
parents: 6
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_7"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_6"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 8
class_name: "ShuffledRowRDD"
repr: "ShuffledRowRDD[8] at javaRDD at ExecutionItem.scala:56"
parents: 7
proto {
name: "my_input_karps_merged_agg/ShuffledRowRDD_8"
op: "ShuffledRowRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_7"
attr {
key: "name"
value {
s: "ShuffledRowRDD"
}
}
}
}
rdd_info {
rdd_id: 9
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[9] at javaRDD at ExecutionItem.scala:56"
parents: 8
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_9"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/ShuffledRowRDD_8"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 10
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[10] at javaRDD at ExecutionItem.scala:56"
parents: 9
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_10"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_9"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 11
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[11] at javaRDD at ExecutionItem.scala:56"
parents: 10
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_11"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_10"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 12
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[12] at createDataFrame at ExecutionItem.scala:56"
parents: 11
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_12"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_11"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 13
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[13] at execute at ExecutionItem.scala:89"
parents: 12
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_13"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_12"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#128L, _2#129]"
proto {
name: "my_input_karps_merged_agg/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#128L,_2#129]"
proto {
name: "my_input_karps_merged_agg/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
}
}
}
}
}
04:14:25 DEBUG:channel: received result for /my_input_karps_merged_agg: local_path {
path: "my_input_karps_merged_agg"
}
status: FINISHED_SUCCESS
final_result {
cell {
struct_value {
values {
int_value: 2
}
values {
double_value: 3.0
}
}
}
cell_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: INT
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
spark_stats {
rdd_info {
rdd_id: 6
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[6] at javaRDD at ExecutionItem.scala:56"
parents: 4
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_6"
op: "MapPartitionsRDD"
input: "my_input/MapPartitionsRDD_4"
input: "^my_input/MapPartitionsRDD_5"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 7
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[7] at javaRDD at ExecutionItem.scala:56"
parents: 6
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_7"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_6"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 8
class_name: "ShuffledRowRDD"
repr: "ShuffledRowRDD[8] at javaRDD at ExecutionItem.scala:56"
parents: 7
proto {
name: "my_input_karps_merged_agg/ShuffledRowRDD_8"
op: "ShuffledRowRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_7"
attr {
key: "name"
value {
s: "ShuffledRowRDD"
}
}
}
}
rdd_info {
rdd_id: 9
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[9] at javaRDD at ExecutionItem.scala:56"
parents: 8
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_9"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/ShuffledRowRDD_8"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 10
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[10] at javaRDD at ExecutionItem.scala:56"
parents: 9
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_10"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_9"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 11
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[11] at javaRDD at ExecutionItem.scala:56"
parents: 10
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_11"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_10"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 12
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[12] at createDataFrame at ExecutionItem.scala:56"
parents: 11
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_12"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_11"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 13
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[13] at execute at ExecutionItem.scala:89"
parents: 12
proto {
name: "my_input_karps_merged_agg/MapPartitionsRDD_13"
op: "MapPartitionsRDD"
input: "my_input_karps_merged_agg/MapPartitionsRDD_12"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#128L, _2#129]"
proto {
name: "my_input_karps_merged_agg/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#128L, _2#129]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#128L,_2#129]"
proto {
name: "my_input_karps_merged_agg/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: long (nullable = false)\n |-- _2: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#128L,_2#129]"
}
}
}
}
}
04:14:25 DEBUG:Calling _progress
04:14:25 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "count1"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 14
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[14] at javaRDD at ExecutionItem.scala:56"
proto {
name: "count1/ParallelCollectionRDD_14"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 15
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[15] at javaRDD at ExecutionItem.scala:56"
parents: 14
proto {
name: "count1/MapPartitionsRDD_15"
op: "MapPartitionsRDD"
input: "count1/ParallelCollectionRDD_14"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 16
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[16] at javaRDD at ExecutionItem.scala:56"
parents: 15
proto {
name: "count1/MapPartitionsRDD_16"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_15"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 17
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[17] at javaRDD at ExecutionItem.scala:56"
parents: 16
proto {
name: "count1/MapPartitionsRDD_17"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_16"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 18
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[18] at createDataFrame at ExecutionItem.scala:56"
parents: 17
proto {
name: "count1/MapPartitionsRDD_18"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_17"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 19
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[19] at execute at ExecutionItem.scala:89"
parents: 18
proto {
name: "count1/MapPartitionsRDD_19"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_18"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#148]"
proto {
name: "count1/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#148]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#148]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#148]"
proto {
name: "count1/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#148]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#148]"
}
}
}
}
}
}
}
04:14:25 DEBUG:channel: received result for /count1: local_path {
path: "count1"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 14
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[14] at javaRDD at ExecutionItem.scala:56"
proto {
name: "count1/ParallelCollectionRDD_14"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 15
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[15] at javaRDD at ExecutionItem.scala:56"
parents: 14
proto {
name: "count1/MapPartitionsRDD_15"
op: "MapPartitionsRDD"
input: "count1/ParallelCollectionRDD_14"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 16
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[16] at javaRDD at ExecutionItem.scala:56"
parents: 15
proto {
name: "count1/MapPartitionsRDD_16"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_15"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 17
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[17] at javaRDD at ExecutionItem.scala:56"
parents: 16
proto {
name: "count1/MapPartitionsRDD_17"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_16"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 18
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[18] at createDataFrame at ExecutionItem.scala:56"
parents: 17
proto {
name: "count1/MapPartitionsRDD_18"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_17"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 19
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[19] at execute at ExecutionItem.scala:89"
parents: 18
proto {
name: "count1/MapPartitionsRDD_19"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_18"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#148]"
proto {
name: "count1/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#148]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#148]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#148]"
proto {
name: "count1/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#148]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#148]"
}
}
}
}
}
04:14:25 DEBUG:Calling _progress
04:14:28 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "count1"
}
status: FINISHED_SUCCESS
final_result {
cell {
int_value: 2
}
cell_type {
basic_type: INT
}
}
spark_stats {
rdd_info {
rdd_id: 14
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[14] at javaRDD at ExecutionItem.scala:56"
proto {
name: "count1/ParallelCollectionRDD_14"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 15
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[15] at javaRDD at ExecutionItem.scala:56"
parents: 14
proto {
name: "count1/MapPartitionsRDD_15"
op: "MapPartitionsRDD"
input: "count1/ParallelCollectionRDD_14"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 16
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[16] at javaRDD at ExecutionItem.scala:56"
parents: 15
proto {
name: "count1/MapPartitionsRDD_16"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_15"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 17
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[17] at javaRDD at ExecutionItem.scala:56"
parents: 16
proto {
name: "count1/MapPartitionsRDD_17"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_16"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 18
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[18] at createDataFrame at ExecutionItem.scala:56"
parents: 17
proto {
name: "count1/MapPartitionsRDD_18"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_17"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 19
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[19] at execute at ExecutionItem.scala:89"
parents: 18
proto {
name: "count1/MapPartitionsRDD_19"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_18"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#148]"
proto {
name: "count1/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#148]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#148]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#148]"
proto {
name: "count1/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#148]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#148]"
}
}
}
}
}
}
}
04:14:28 DEBUG:channel: received result for /count1: local_path {
path: "count1"
}
status: FINISHED_SUCCESS
final_result {
cell {
int_value: 2
}
cell_type {
basic_type: INT
}
}
spark_stats {
rdd_info {
rdd_id: 14
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[14] at javaRDD at ExecutionItem.scala:56"
proto {
name: "count1/ParallelCollectionRDD_14"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 15
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[15] at javaRDD at ExecutionItem.scala:56"
parents: 14
proto {
name: "count1/MapPartitionsRDD_15"
op: "MapPartitionsRDD"
input: "count1/ParallelCollectionRDD_14"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 16
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[16] at javaRDD at ExecutionItem.scala:56"
parents: 15
proto {
name: "count1/MapPartitionsRDD_16"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_15"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 17
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[17] at javaRDD at ExecutionItem.scala:56"
parents: 16
proto {
name: "count1/MapPartitionsRDD_17"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_16"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 18
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[18] at createDataFrame at ExecutionItem.scala:56"
parents: 17
proto {
name: "count1/MapPartitionsRDD_18"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_17"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 19
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[19] at execute at ExecutionItem.scala:89"
parents: 18
proto {
name: "count1/MapPartitionsRDD_19"
op: "MapPartitionsRDD"
input: "count1/MapPartitionsRDD_18"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#148]"
proto {
name: "count1/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#148]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#148]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#148]"
proto {
name: "count1/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: integer (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#148]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#148]"
}
}
}
}
}
04:14:28 DEBUG:Calling _progress
04:14:28 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "sum0"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 20
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[20] at javaRDD at ExecutionItem.scala:56"
proto {
name: "sum0/ParallelCollectionRDD_20"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 21
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[21] at javaRDD at ExecutionItem.scala:56"
parents: 20
proto {
name: "sum0/MapPartitionsRDD_21"
op: "MapPartitionsRDD"
input: "sum0/ParallelCollectionRDD_20"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 22
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[22] at javaRDD at ExecutionItem.scala:56"
parents: 21
proto {
name: "sum0/MapPartitionsRDD_22"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_21"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 23
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[23] at javaRDD at ExecutionItem.scala:56"
parents: 22
proto {
name: "sum0/MapPartitionsRDD_23"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_22"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 24
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[24] at createDataFrame at ExecutionItem.scala:56"
parents: 23
proto {
name: "sum0/MapPartitionsRDD_24"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_23"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 25
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[25] at execute at ExecutionItem.scala:89"
parents: 24
proto {
name: "sum0/MapPartitionsRDD_25"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_24"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#166]"
proto {
name: "sum0/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#166]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#166]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#166]"
proto {
name: "sum0/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#166]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#166]"
}
}
}
}
}
}
}
04:14:28 DEBUG:channel: received result for /sum0: local_path {
path: "sum0"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 20
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[20] at javaRDD at ExecutionItem.scala:56"
proto {
name: "sum0/ParallelCollectionRDD_20"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 21
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[21] at javaRDD at ExecutionItem.scala:56"
parents: 20
proto {
name: "sum0/MapPartitionsRDD_21"
op: "MapPartitionsRDD"
input: "sum0/ParallelCollectionRDD_20"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 22
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[22] at javaRDD at ExecutionItem.scala:56"
parents: 21
proto {
name: "sum0/MapPartitionsRDD_22"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_21"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 23
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[23] at javaRDD at ExecutionItem.scala:56"
parents: 22
proto {
name: "sum0/MapPartitionsRDD_23"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_22"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 24
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[24] at createDataFrame at ExecutionItem.scala:56"
parents: 23
proto {
name: "sum0/MapPartitionsRDD_24"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_23"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 25
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[25] at execute at ExecutionItem.scala:89"
parents: 24
proto {
name: "sum0/MapPartitionsRDD_25"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_24"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#166]"
proto {
name: "sum0/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#166]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#166]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#166]"
proto {
name: "sum0/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#166]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#166]"
}
}
}
}
}
04:14:28 DEBUG:Calling _progress
04:14:28 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "sum0"
}
status: FINISHED_SUCCESS
final_result {
cell {
double_value: 3.0
}
cell_type {
basic_type: DOUBLE
}
}
spark_stats {
rdd_info {
rdd_id: 20
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[20] at javaRDD at ExecutionItem.scala:56"
proto {
name: "sum0/ParallelCollectionRDD_20"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 21
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[21] at javaRDD at ExecutionItem.scala:56"
parents: 20
proto {
name: "sum0/MapPartitionsRDD_21"
op: "MapPartitionsRDD"
input: "sum0/ParallelCollectionRDD_20"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 22
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[22] at javaRDD at ExecutionItem.scala:56"
parents: 21
proto {
name: "sum0/MapPartitionsRDD_22"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_21"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 23
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[23] at javaRDD at ExecutionItem.scala:56"
parents: 22
proto {
name: "sum0/MapPartitionsRDD_23"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_22"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 24
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[24] at createDataFrame at ExecutionItem.scala:56"
parents: 23
proto {
name: "sum0/MapPartitionsRDD_24"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_23"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 25
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[25] at execute at ExecutionItem.scala:89"
parents: 24
proto {
name: "sum0/MapPartitionsRDD_25"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_24"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#166]"
proto {
name: "sum0/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#166]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#166]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#166]"
proto {
name: "sum0/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#166]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#166]"
}
}
}
}
}
}
}
04:14:28 DEBUG:channel: received result for /sum0: local_path {
path: "sum0"
}
status: FINISHED_SUCCESS
final_result {
cell {
double_value: 3.0
}
cell_type {
basic_type: DOUBLE
}
}
spark_stats {
rdd_info {
rdd_id: 20
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[20] at javaRDD at ExecutionItem.scala:56"
proto {
name: "sum0/ParallelCollectionRDD_20"
op: "ParallelCollectionRDD"
input: "^my_input_karps_merged_agg/MapPartitionsRDD_13"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 21
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[21] at javaRDD at ExecutionItem.scala:56"
parents: 20
proto {
name: "sum0/MapPartitionsRDD_21"
op: "MapPartitionsRDD"
input: "sum0/ParallelCollectionRDD_20"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 22
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[22] at javaRDD at ExecutionItem.scala:56"
parents: 21
proto {
name: "sum0/MapPartitionsRDD_22"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_21"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 23
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[23] at javaRDD at ExecutionItem.scala:56"
parents: 22
proto {
name: "sum0/MapPartitionsRDD_23"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_22"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 24
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[24] at createDataFrame at ExecutionItem.scala:56"
parents: 23
proto {
name: "sum0/MapPartitionsRDD_24"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_23"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 25
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[25] at execute at ExecutionItem.scala:89"
parents: 24
proto {
name: "sum0/MapPartitionsRDD_25"
op: "MapPartitionsRDD"
input: "sum0/MapPartitionsRDD_24"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#166]"
proto {
name: "sum0/LogicalRDD_0"
op: "LogicalRDD"
input: "^my_input_karps_merged_agg/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#166]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#166]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#166]"
proto {
name: "sum0/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^my_input_karps_merged_agg/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#166]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#166]"
}
}
}
}
}
04:14:28 DEBUG:Calling _progress
04:14:28 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "cast_double2"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 26
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[26] at javaRDD at ExecutionItem.scala:56"
proto {
name: "cast_double2/ParallelCollectionRDD_26"
op: "ParallelCollectionRDD"
input: "^count1/MapPartitionsRDD_19"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 27
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[27] at javaRDD at ExecutionItem.scala:56"
parents: 26
proto {
name: "cast_double2/MapPartitionsRDD_27"
op: "MapPartitionsRDD"
input: "cast_double2/ParallelCollectionRDD_26"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 28
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[28] at javaRDD at ExecutionItem.scala:56"
parents: 27
proto {
name: "cast_double2/MapPartitionsRDD_28"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_27"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 29
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[29] at javaRDD at ExecutionItem.scala:56"
parents: 28
proto {
name: "cast_double2/MapPartitionsRDD_29"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_28"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 30
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[30] at createDataFrame at ExecutionItem.scala:56"
parents: 29
proto {
name: "cast_double2/MapPartitionsRDD_30"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_29"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 31
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[31] at execute at ExecutionItem.scala:89"
parents: 30
proto {
name: "cast_double2/MapPartitionsRDD_31"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_30"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#185]"
proto {
name: "cast_double2/LogicalRDD_0"
op: "LogicalRDD"
input: "^count1/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#185]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#185]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#185]"
proto {
name: "cast_double2/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^count1/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#185]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#185]"
}
}
}
}
}
}
}
04:14:28 DEBUG:channel: received result for /cast_double2: local_path {
path: "cast_double2"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 26
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[26] at javaRDD at ExecutionItem.scala:56"
proto {
name: "cast_double2/ParallelCollectionRDD_26"
op: "ParallelCollectionRDD"
input: "^count1/MapPartitionsRDD_19"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 27
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[27] at javaRDD at ExecutionItem.scala:56"
parents: 26
proto {
name: "cast_double2/MapPartitionsRDD_27"
op: "MapPartitionsRDD"
input: "cast_double2/ParallelCollectionRDD_26"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 28
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[28] at javaRDD at ExecutionItem.scala:56"
parents: 27
proto {
name: "cast_double2/MapPartitionsRDD_28"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_27"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 29
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[29] at javaRDD at ExecutionItem.scala:56"
parents: 28
proto {
name: "cast_double2/MapPartitionsRDD_29"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_28"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 30
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[30] at createDataFrame at ExecutionItem.scala:56"
parents: 29
proto {
name: "cast_double2/MapPartitionsRDD_30"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_29"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 31
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[31] at execute at ExecutionItem.scala:89"
parents: 30
proto {
name: "cast_double2/MapPartitionsRDD_31"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_30"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#185]"
proto {
name: "cast_double2/LogicalRDD_0"
op: "LogicalRDD"
input: "^count1/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#185]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#185]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#185]"
proto {
name: "cast_double2/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^count1/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#185]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#185]"
}
}
}
}
}
04:14:28 DEBUG:Calling _progress
04:14:31 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "cast_double2"
}
status: FINISHED_SUCCESS
final_result {
cell {
double_value: 2.0
}
cell_type {
basic_type: DOUBLE
}
}
spark_stats {
rdd_info {
rdd_id: 26
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[26] at javaRDD at ExecutionItem.scala:56"
proto {
name: "cast_double2/ParallelCollectionRDD_26"
op: "ParallelCollectionRDD"
input: "^count1/MapPartitionsRDD_19"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 27
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[27] at javaRDD at ExecutionItem.scala:56"
parents: 26
proto {
name: "cast_double2/MapPartitionsRDD_27"
op: "MapPartitionsRDD"
input: "cast_double2/ParallelCollectionRDD_26"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 28
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[28] at javaRDD at ExecutionItem.scala:56"
parents: 27
proto {
name: "cast_double2/MapPartitionsRDD_28"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_27"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 29
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[29] at javaRDD at ExecutionItem.scala:56"
parents: 28
proto {
name: "cast_double2/MapPartitionsRDD_29"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_28"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 30
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[30] at createDataFrame at ExecutionItem.scala:56"
parents: 29
proto {
name: "cast_double2/MapPartitionsRDD_30"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_29"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 31
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[31] at execute at ExecutionItem.scala:89"
parents: 30
proto {
name: "cast_double2/MapPartitionsRDD_31"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_30"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#185]"
proto {
name: "cast_double2/LogicalRDD_0"
op: "LogicalRDD"
input: "^count1/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#185]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#185]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#185]"
proto {
name: "cast_double2/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^count1/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#185]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#185]"
}
}
}
}
}
}
}
04:14:31 DEBUG:channel: received result for /cast_double2: local_path {
path: "cast_double2"
}
status: FINISHED_SUCCESS
final_result {
cell {
double_value: 2.0
}
cell_type {
basic_type: DOUBLE
}
}
spark_stats {
rdd_info {
rdd_id: 26
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[26] at javaRDD at ExecutionItem.scala:56"
proto {
name: "cast_double2/ParallelCollectionRDD_26"
op: "ParallelCollectionRDD"
input: "^count1/MapPartitionsRDD_19"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 27
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[27] at javaRDD at ExecutionItem.scala:56"
parents: 26
proto {
name: "cast_double2/MapPartitionsRDD_27"
op: "MapPartitionsRDD"
input: "cast_double2/ParallelCollectionRDD_26"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 28
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[28] at javaRDD at ExecutionItem.scala:56"
parents: 27
proto {
name: "cast_double2/MapPartitionsRDD_28"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_27"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 29
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[29] at javaRDD at ExecutionItem.scala:56"
parents: 28
proto {
name: "cast_double2/MapPartitionsRDD_29"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_28"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 30
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[30] at createDataFrame at ExecutionItem.scala:56"
parents: 29
proto {
name: "cast_double2/MapPartitionsRDD_30"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_29"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 31
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[31] at execute at ExecutionItem.scala:89"
parents: 30
proto {
name: "cast_double2/MapPartitionsRDD_31"
op: "MapPartitionsRDD"
input: "cast_double2/MapPartitionsRDD_30"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#185]"
proto {
name: "cast_double2/LogicalRDD_0"
op: "LogicalRDD"
input: "^count1/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#185]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#185]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#185]"
proto {
name: "cast_double2/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^count1/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#185]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#185]"
}
}
}
}
}
04:14:31 DEBUG:Calling _progress
04:14:31 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "divide3_karps_localpack"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 32
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[32] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3_karps_localpack/ParallelCollectionRDD_32"
op: "ParallelCollectionRDD"
input: "^sum0/MapPartitionsRDD_25"
input: "^cast_double2/MapPartitionsRDD_31"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 33
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[33] at javaRDD at ExecutionItem.scala:56"
parents: 32
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_33"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/ParallelCollectionRDD_32"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 34
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[34] at javaRDD at ExecutionItem.scala:56"
parents: 33
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_34"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_33"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 35
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[35] at javaRDD at ExecutionItem.scala:56"
parents: 34
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_35"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_34"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 36
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[36] at createDataFrame at ExecutionItem.scala:56"
parents: 35
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_36"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_35"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 37
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[37] at execute at ExecutionItem.scala:89"
parents: 36
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_37"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_36"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#198, _2#199]"
proto {
name: "divide3_karps_localpack/LogicalRDD_0"
op: "LogicalRDD"
input: "^sum0/LogicalRDD_0"
input: "^cast_double2/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#198,_2#199]"
proto {
name: "divide3_karps_localpack/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^sum0/Scan_ExistingRDD_0"
input: "^cast_double2/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
}
}
}
}
}
04:14:31 DEBUG:channel: received result for /divide3_karps_localpack: local_path {
path: "divide3_karps_localpack"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 32
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[32] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3_karps_localpack/ParallelCollectionRDD_32"
op: "ParallelCollectionRDD"
input: "^sum0/MapPartitionsRDD_25"
input: "^cast_double2/MapPartitionsRDD_31"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 33
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[33] at javaRDD at ExecutionItem.scala:56"
parents: 32
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_33"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/ParallelCollectionRDD_32"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 34
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[34] at javaRDD at ExecutionItem.scala:56"
parents: 33
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_34"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_33"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 35
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[35] at javaRDD at ExecutionItem.scala:56"
parents: 34
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_35"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_34"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 36
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[36] at createDataFrame at ExecutionItem.scala:56"
parents: 35
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_36"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_35"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 37
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[37] at execute at ExecutionItem.scala:89"
parents: 36
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_37"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_36"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#198, _2#199]"
proto {
name: "divide3_karps_localpack/LogicalRDD_0"
op: "LogicalRDD"
input: "^sum0/LogicalRDD_0"
input: "^cast_double2/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#198,_2#199]"
proto {
name: "divide3_karps_localpack/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^sum0/Scan_ExistingRDD_0"
input: "^cast_double2/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
}
}
}
04:14:31 DEBUG:Calling _progress
04:14:31 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "divide3_karps_localpack"
}
status: FINISHED_SUCCESS
final_result {
cell {
struct_value {
values {
double_value: 3.0
}
values {
double_value: 2.0
}
}
}
cell_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
spark_stats {
rdd_info {
rdd_id: 32
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[32] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3_karps_localpack/ParallelCollectionRDD_32"
op: "ParallelCollectionRDD"
input: "^sum0/MapPartitionsRDD_25"
input: "^cast_double2/MapPartitionsRDD_31"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 33
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[33] at javaRDD at ExecutionItem.scala:56"
parents: 32
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_33"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/ParallelCollectionRDD_32"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 34
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[34] at javaRDD at ExecutionItem.scala:56"
parents: 33
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_34"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_33"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 35
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[35] at javaRDD at ExecutionItem.scala:56"
parents: 34
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_35"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_34"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 36
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[36] at createDataFrame at ExecutionItem.scala:56"
parents: 35
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_36"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_35"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 37
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[37] at execute at ExecutionItem.scala:89"
parents: 36
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_37"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_36"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#198, _2#199]"
proto {
name: "divide3_karps_localpack/LogicalRDD_0"
op: "LogicalRDD"
input: "^sum0/LogicalRDD_0"
input: "^cast_double2/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#198,_2#199]"
proto {
name: "divide3_karps_localpack/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^sum0/Scan_ExistingRDD_0"
input: "^cast_double2/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
}
}
}
}
}
04:14:31 DEBUG:channel: received result for /divide3_karps_localpack: local_path {
path: "divide3_karps_localpack"
}
status: FINISHED_SUCCESS
final_result {
cell {
struct_value {
values {
double_value: 3.0
}
values {
double_value: 2.0
}
}
}
cell_type {
struct_type {
fields {
field_name: "_1"
field_type {
basic_type: DOUBLE
}
}
fields {
field_name: "_2"
field_type {
basic_type: DOUBLE
}
}
}
}
}
spark_stats {
rdd_info {
rdd_id: 32
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[32] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3_karps_localpack/ParallelCollectionRDD_32"
op: "ParallelCollectionRDD"
input: "^sum0/MapPartitionsRDD_25"
input: "^cast_double2/MapPartitionsRDD_31"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 33
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[33] at javaRDD at ExecutionItem.scala:56"
parents: 32
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_33"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/ParallelCollectionRDD_32"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 34
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[34] at javaRDD at ExecutionItem.scala:56"
parents: 33
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_34"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_33"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 35
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[35] at javaRDD at ExecutionItem.scala:56"
parents: 34
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_35"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_34"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 36
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[36] at createDataFrame at ExecutionItem.scala:56"
parents: 35
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_36"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_35"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 37
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[37] at execute at ExecutionItem.scala:89"
parents: 36
proto {
name: "divide3_karps_localpack/MapPartitionsRDD_37"
op: "MapPartitionsRDD"
input: "divide3_karps_localpack/MapPartitionsRDD_36"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [_1#198, _2#199]"
proto {
name: "divide3_karps_localpack/LogicalRDD_0"
op: "LogicalRDD"
input: "^sum0/LogicalRDD_0"
input: "^cast_double2/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [_1#198, _2#199]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[_1#198,_2#199]"
proto {
name: "divide3_karps_localpack/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^sum0/Scan_ExistingRDD_0"
input: "^cast_double2/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- _1: double (nullable = false)\n |-- _2: double (nullable = false)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[_1#198,_2#199]"
}
}
}
}
}
04:14:31 DEBUG:Calling _progress
04:14:31 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "divide3"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 38
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[38] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3/ParallelCollectionRDD_38"
op: "ParallelCollectionRDD"
input: "^divide3_karps_localpack/MapPartitionsRDD_37"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 39
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[39] at javaRDD at ExecutionItem.scala:56"
parents: 38
proto {
name: "divide3/MapPartitionsRDD_39"
op: "MapPartitionsRDD"
input: "divide3/ParallelCollectionRDD_38"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 40
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[40] at javaRDD at ExecutionItem.scala:56"
parents: 39
proto {
name: "divide3/MapPartitionsRDD_40"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_39"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 41
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[41] at javaRDD at ExecutionItem.scala:56"
parents: 40
proto {
name: "divide3/MapPartitionsRDD_41"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_40"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 42
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[42] at createDataFrame at ExecutionItem.scala:56"
parents: 41
proto {
name: "divide3/MapPartitionsRDD_42"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_41"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 43
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[43] at execute at ExecutionItem.scala:89"
parents: 42
proto {
name: "divide3/MapPartitionsRDD_43"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_42"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#221]"
proto {
name: "divide3/LogicalRDD_0"
op: "LogicalRDD"
input: "^divide3_karps_localpack/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#221]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#221]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#221]"
proto {
name: "divide3/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^divide3_karps_localpack/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#221]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#221]"
}
}
}
}
}
}
}
04:14:31 DEBUG:channel: received result for /divide3: local_path {
path: "divide3"
}
status: SCHEDULED
spark_stats {
rdd_info {
rdd_id: 38
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[38] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3/ParallelCollectionRDD_38"
op: "ParallelCollectionRDD"
input: "^divide3_karps_localpack/MapPartitionsRDD_37"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 39
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[39] at javaRDD at ExecutionItem.scala:56"
parents: 38
proto {
name: "divide3/MapPartitionsRDD_39"
op: "MapPartitionsRDD"
input: "divide3/ParallelCollectionRDD_38"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 40
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[40] at javaRDD at ExecutionItem.scala:56"
parents: 39
proto {
name: "divide3/MapPartitionsRDD_40"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_39"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 41
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[41] at javaRDD at ExecutionItem.scala:56"
parents: 40
proto {
name: "divide3/MapPartitionsRDD_41"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_40"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 42
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[42] at createDataFrame at ExecutionItem.scala:56"
parents: 41
proto {
name: "divide3/MapPartitionsRDD_42"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_41"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 43
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[43] at execute at ExecutionItem.scala:89"
parents: 42
proto {
name: "divide3/MapPartitionsRDD_43"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_42"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#221]"
proto {
name: "divide3/LogicalRDD_0"
op: "LogicalRDD"
input: "^divide3_karps_localpack/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#221]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#221]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#221]"
proto {
name: "divide3/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^divide3_karps_localpack/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#221]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#221]"
}
}
}
}
}
04:14:31 DEBUG:Calling _progress
04:14:32 DEBUG:channel: got value <class 'karps.proto.interface_pb2.ComputationStreamResponse'>: session {
id: "test3"
}
computation {
id: "1"
}
results {
results {
local_path {
path: "divide3"
}
status: FINISHED_SUCCESS
final_result {
cell {
double_value: 1.5
}
cell_type {
basic_type: DOUBLE
}
}
spark_stats {
rdd_info {
rdd_id: 38
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[38] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3/ParallelCollectionRDD_38"
op: "ParallelCollectionRDD"
input: "^divide3_karps_localpack/MapPartitionsRDD_37"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 39
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[39] at javaRDD at ExecutionItem.scala:56"
parents: 38
proto {
name: "divide3/MapPartitionsRDD_39"
op: "MapPartitionsRDD"
input: "divide3/ParallelCollectionRDD_38"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 40
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[40] at javaRDD at ExecutionItem.scala:56"
parents: 39
proto {
name: "divide3/MapPartitionsRDD_40"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_39"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 41
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[41] at javaRDD at ExecutionItem.scala:56"
parents: 40
proto {
name: "divide3/MapPartitionsRDD_41"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_40"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 42
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[42] at createDataFrame at ExecutionItem.scala:56"
parents: 41
proto {
name: "divide3/MapPartitionsRDD_42"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_41"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 43
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[43] at execute at ExecutionItem.scala:89"
parents: 42
proto {
name: "divide3/MapPartitionsRDD_43"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_42"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#221]"
proto {
name: "divide3/LogicalRDD_0"
op: "LogicalRDD"
input: "^divide3_karps_localpack/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#221]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#221]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#221]"
proto {
name: "divide3/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^divide3_karps_localpack/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#221]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#221]"
}
}
}
}
}
}
}
04:14:32 DEBUG:channel: received result for /divide3: local_path {
path: "divide3"
}
status: FINISHED_SUCCESS
final_result {
cell {
double_value: 1.5
}
cell_type {
basic_type: DOUBLE
}
}
spark_stats {
rdd_info {
rdd_id: 38
class_name: "ParallelCollectionRDD"
repr: "ParallelCollectionRDD[38] at javaRDD at ExecutionItem.scala:56"
proto {
name: "divide3/ParallelCollectionRDD_38"
op: "ParallelCollectionRDD"
input: "^divide3_karps_localpack/MapPartitionsRDD_37"
attr {
key: "name"
value {
s: "ParallelCollectionRDD"
}
}
}
}
rdd_info {
rdd_id: 39
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[39] at javaRDD at ExecutionItem.scala:56"
parents: 38
proto {
name: "divide3/MapPartitionsRDD_39"
op: "MapPartitionsRDD"
input: "divide3/ParallelCollectionRDD_38"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 40
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[40] at javaRDD at ExecutionItem.scala:56"
parents: 39
proto {
name: "divide3/MapPartitionsRDD_40"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_39"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 41
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[41] at javaRDD at ExecutionItem.scala:56"
parents: 40
proto {
name: "divide3/MapPartitionsRDD_41"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_40"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 42
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[42] at createDataFrame at ExecutionItem.scala:56"
parents: 41
proto {
name: "divide3/MapPartitionsRDD_42"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_41"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
rdd_info {
rdd_id: 43
class_name: "MapPartitionsRDD"
repr: "MapPartitionsRDD[43] at execute at ExecutionItem.scala:89"
parents: 42
proto {
name: "divide3/MapPartitionsRDD_43"
op: "MapPartitionsRDD"
input: "divide3/MapPartitionsRDD_42"
attr {
key: "name"
value {
s: "MapPartitionsRDD"
}
}
}
}
parsed {
node_id: "LogicalRDD_0"
full_name: "LogicalRDD [value#221]"
proto {
name: "divide3/LogicalRDD_0"
op: "LogicalRDD"
input: "^divide3_karps_localpack/LogicalRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "LogicalRDD [value#221]"
}
}
attr {
key: "verbose"
value {
s: "LogicalRDD [value#221]"
}
}
}
}
physical {
node_id: "Scan_ExistingRDD_0"
full_name: "Scan ExistingRDD[value#221]"
proto {
name: "divide3/Scan_ExistingRDD_0"
op: "Scan_ExistingRDD"
input: "^divide3_karps_localpack/Scan_ExistingRDD_0"
attr {
key: "schema"
value {
s: "root\n |-- value: double (nullable = true)\n"
}
}
attr {
key: "simple"
value {
s: "Scan ExistingRDD[value#221]"
}
}
attr {
key: "verbose"
value {
s: "Scan ExistingRDD[value#221]"
}
}
}
}
}
Out[11]:
(double, double_value: 1.5
)
In [12]:
def my_mean(df):
cached_df = f.autocache(df)
In [13]:
type(my_mean)
Out[13]:
function
In [14]:
from tensorflow.core.framework import graph_pb2
In [15]:
graph_pb2.GraphDef()
Out[15]:
In [16]:
_17.node.add?
Object `_17.node.add` not found.
In [ ]:
Content source: tjhunter/karps
Similar notebooks: