In [1]:

    

:load KrapshDisplays KrapshDagDisplay


    

In [2]:

    

:extension DeriveGeneric
:extension FlexibleContexts
:extension OverloadedStrings
:extension GeneralizedNewtypeDeriving
:extension FlexibleInstances
:extension MultiParamTypeClasses


    

In [3]:

    

import Spark.Core.Dataset
import Spark.Core.Context
import Spark.Core.Column
import Spark.Core.ColumnFunctions
import Spark.Core.Functions
import Spark.Core.Row
import Spark.Core.Types
import Spark.Core.Try

import qualified Data.Vector as V
import qualified Data.Text as T
import GHC.Generics
import IHaskell.Display


    

In [4]:

    

import Spark.Core.StructuresInternal(ComputationID(..))


    

In [5]:

    

conf = defaultConf {
        confEndPoint = "http://localhost",
        confRequestedSessionName = "cache_sum1" }

createSparkSessionDef conf


    

    






[Debug] Creating spark session at url: http://localhost:8081/sessions/cache_sum1 @(<unknown>:<unknown> <unknown>:0:0)

In [6]:

    

stats <- computationStatsDef (ComputationID "0")


    

    






[Debug] computationStats: stats for ComputationID {unComputationID = "0"} @(<unknown>:<unknown> <unknown>:0:0)
[Debug] Sending computations stats request at url: http://localhost:8081/computations_status/cache_sum1/0/ @(<unknown>:<unknown> <unknown>:0:0)

In [7]:

    

let ns = statsToExportNodes stats


    

In [8]:

    

let c = T.unpack (tfIFrame ns)
Display [html c]


    

In [19]:

    

import qualified Data.ByteString.Char8 as C8
import Spark.Core.Internal.DAGFunctions(pruneLexicographic)
import Spark.Core.Internal.DAGStructures

let vxId = VertexId . C8.pack . T.unpack

let f en = (vxId (enName en), vxId <$> enDeps en, en)
let ns2 = f <$> ns
--pruneLexicographic ns2
[n <- enName <$> ns | ]


    

    





Couldn't match expected type `VertexId' with actual type `[(VertexId, [VertexId], ExportNode)]'
In the first argument of `pruneLexicographic', namely `ns2'
In the expression: pruneLexicographic ns2

In [13]:

    

import Spark.Core.Internal.DAGFunctions(pruneLexicographic)


    

In [9]:

    

ns


    

    






[ExportNode {enName = "select_2ba89f/ParallelCollectionRDD-2", enOp = "ParallelCollectionRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","ParallelCollectionRDD[2] at persist at SparkRegistry.scala:189")]},ExportNode {enName = "select_2ba89f/MapPartitionsRDD-3", enOp = "MapPartitionsRDD", enDeps = ["select_2ba89f/ParallelCollectionRDD-2"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[3] at persist at SparkRegistry.scala:189")]},ExportNode {enName = "select_2ba89f/MapPartitionsRDD-4", enOp = "MapPartitionsRDD", enDeps = ["select_2ba89f/MapPartitionsRDD-3"], enLogicalDeps = [], enAttributes = [("name","LocalTableScan [_1#0]\n MapPartitionsRDD[4] at persist at SparkRegistry.scala:189")]},ExportNode {enName = "select_2ba89f/MapPartitionsRDD-12", enOp = "MapPartitionsRDD", enDeps = ["select_2ba89f/MapPartitionsRDD-4"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[12] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "distributedliteral_1ba31e/ParallelCollectionRDD-0", enOp = "ParallelCollectionRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","ParallelCollectionRDD[0] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "distributedliteral_1ba31e/MapPartitionsRDD-1", enOp = "MapPartitionsRDD", enDeps = ["distributedliteral_1ba31e/ParallelCollectionRDD-0","unpersist_5f85cd/ParallelCollectionRDD-0"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[1] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "count_abc591/MapPartitionsRDD-47", enOp = "MapPartitionsRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[47] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "count_abc591/MapPartitionsRDD-48", enOp = "MapPartitionsRDD", enDeps = ["count_abc591/MapPartitionsRDD-47"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[48] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "count_abc591/MapPartitionsRDD-49", enOp = "MapPartitionsRDD", enDeps = ["count_abc591/MapPartitionsRDD-48"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[49] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "count_abc591/ShuffledRowRDD-50", enOp = "ShuffledRowRDD", enDeps = ["count_abc591/MapPartitionsRDD-49"], enLogicalDeps = [], enAttributes = [("name","ShuffledRowRDD[50] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "count_abc591/MapPartitionsRDD-51", enOp = "MapPartitionsRDD", enDeps = ["count_abc591/ShuffledRowRDD-50"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[51] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "autocache_70493c/MapPartitionsRDD-11", enOp = "MapPartitionsRDD", enDeps = ["distributedliteral_1ba31e/ParallelCollectionRDD-0","unpersist_5f85cd/ParallelCollectionRDD-0"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[11] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "sum_d52323/MapPartitionsRDD-39", enOp = "MapPartitionsRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[39] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "sum_d52323/MapPartitionsRDD-40", enOp = "MapPartitionsRDD", enDeps = ["sum_d52323/MapPartitionsRDD-39"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[40] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "sum_d52323/MapPartitionsRDD-41", enOp = "MapPartitionsRDD", enDeps = ["sum_d52323/MapPartitionsRDD-40"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[41] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "sum_d52323/ShuffledRowRDD-42", enOp = "ShuffledRowRDD", enDeps = ["sum_d52323/MapPartitionsRDD-41"], enLogicalDeps = [], enAttributes = [("name","ShuffledRowRDD[42] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "sum_d52323/MapPartitionsRDD-43", enOp = "MapPartitionsRDD", enDeps = ["sum_d52323/ShuffledRowRDD-42"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[43] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "distributedliteral_1ba31e/ParallelCollectionRDD-0", enOp = "ParallelCollectionRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","ParallelCollectionRDD[0] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "unpersist_5f85cd/MapPartitionsRDD-23", enOp = "MapPartitionsRDD", enDeps = ["distributedliteral_1ba31e/ParallelCollectionRDD-0","unpersist_5f85cd/ParallelCollectionRDD-0"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[23] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "localplus_23aca6/ParallelCollectionRDD-56", enOp = "ParallelCollectionRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","ParallelCollectionRDD[56] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "localplus_23aca6/MapPartitionsRDD-57", enOp = "MapPartitionsRDD", enDeps = ["localplus_23aca6/ParallelCollectionRDD-56"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[57] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "localidentity_21dd12/ParallelCollectionRDD-52", enOp = "ParallelCollectionRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","ParallelCollectionRDD[52] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "localidentity_21dd12/MapPartitionsRDD-53", enOp = "MapPartitionsRDD", enDeps = ["localidentity_21dd12/ParallelCollectionRDD-52"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[53] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "localidentity_5b45bc/ParallelCollectionRDD-54", enOp = "ParallelCollectionRDD", enDeps = [], enLogicalDeps = [], enAttributes = [("name","ParallelCollectionRDD[54] at execute at ExecutionItem.scala:122")]},ExportNode {enName = "localidentity_5b45bc/MapPartitionsRDD-55", enOp = "MapPartitionsRDD", enDeps = ["localidentity_5b45bc/ParallelCollectionRDD-54"], enLogicalDeps = [], enAttributes = [("name","MapPartitionsRDD[55] at execute at ExecutionItem.scala:122")]}]

In [ ]: