In [1]:
import pyspark
from pyspark.mllib import linalg as mllib_linalg
from pyspark.mllib.linalg.distributed import MatrixEntry, CoordinateMatrix
from pyspark.ml import linalg as ml_linalg
from pyspark.ml import feature
from pyspark.sql import functions as F
from pyspark.sql import types as T
from pyspark.ml import Pipeline
In [2]:
import numpy as np
import pandas as pd
from scipy import sparse
/usr/local/lib/python3.5/dist-packages/pandas/core/computation/__init__.py:18: UserWarning: The installed version of numexpr 2.4.3 is not supported in pandas and will be not be used
The minimum supported version is 2.4.6
ver=ver, min_ver=_MIN_NUMEXPR_VERSION), UserWarning)
In [ ]:
In [3]:
n = 150
a = 2.5
z = np.linspace(0,3,n)
z_m = -z
def draw(n, k, draws):
sample = np.random.choice(range(n), replace=False, size=draws).tolist()
return [k if i in sample else float('NAN') for i in range(n)]
def generate_string_data(z,a,k, draws=1):
return list(zip(draw(len(z),float(k),draws), np.array([a*np.sin(z), a*np.cos(z), z]).T))
def generate_springs(n,a,draws=1):
tmp = generate_string_data(z,a,0,draws)
tmp2 = generate_string_data(z_m,a,1,draws)
tmp3 = map(lambda x : (x[0],*x[1]) ,enumerate(tmp+tmp2))
return list(tmp3)
In [4]:
spring_data = generate_springs(n,a,1)
spring_data
Out[4]:
[(0, nan, array([ 0. , 2.5, 0. ])),
(1, nan, array([ 0.05033217, 2.49949328, 0.02013423])),
(2, nan, array([ 0.10064394, 2.49797334, 0.04026846])),
(3, nan, array([ 0.1509149 , 2.49544078, 0.06040268])),
(4, nan, array([ 0.20112469, 2.49189664, 0.08053691])),
(5, nan, array([ 0.25125296, 2.48734235, 0.10067114])),
(6, nan, array([ 0.30127936, 2.48177975, 0.12080537])),
(7, nan, array([ 0.35118364, 2.47521111, 0.1409396 ])),
(8, nan, array([ 0.40094556, 2.46763909, 0.16107383])),
(9, nan, array([ 0.45054495, 2.45906674, 0.18120805])),
(10, nan, array([ 0.49996169, 2.44949756, 0.20134228])),
(11, nan, array([ 0.54917577, 2.43893542, 0.22147651])),
(12, nan, array([ 0.59816722, 2.4273846 , 0.24161074])),
(13, nan, array([ 0.64691619, 2.41484978, 0.26174497])),
(14, nan, array([ 0.69540292, 2.40133604, 0.28187919])),
(15, nan, array([ 0.74360776, 2.38684886, 0.30201342])),
(16, nan, array([ 0.79151115, 2.37139413, 0.32214765])),
(17, nan, array([ 0.83909369, 2.35497809, 0.34228188])),
(18, nan, array([ 0.88633608, 2.3376074 , 0.36241611])),
(19, nan, array([ 0.93321917, 2.31928911, 0.38255034])),
(20, nan, array([ 0.97972396, 2.30003064, 0.40268456])),
(21, nan, array([ 1.02583159, 2.27983981, 0.42281879])),
(22, nan, array([ 1.07152338, 2.25872478, 0.44295302])),
(23, nan, array([ 1.11678081, 2.23669413, 0.46308725])),
(24, nan, array([ 1.16158552, 2.21375678, 0.48322148])),
(25, nan, array([ 1.20591935, 2.18992203, 0.5033557 ])),
(26, nan, array([ 1.24976434, 2.16519955, 0.52348993])),
(27, nan, array([ 1.29310271, 2.13959935, 0.54362416])),
(28, nan, array([ 1.33591688, 2.11313182, 0.56375839])),
(29, nan, array([ 1.37818951, 2.08580768, 0.58389262])),
(30, nan, array([ 1.41990346, 2.057638 , 0.60402685])),
(31, nan, array([ 1.46104182, 2.02863422, 0.62416107])),
(32, nan, array([ 1.50158791, 1.99880808, 0.6442953 ])),
(33, nan, array([ 1.5415253 , 1.96817168, 0.66442953])),
(34, nan, array([ 1.58083779, 1.93673743, 0.68456376])),
(35, nan, array([ 1.61950946, 1.90451808, 0.70469799])),
(36, nan, array([ 1.65752462, 1.87152669, 0.72483221])),
(37, nan, array([ 1.69486786, 1.83777663, 0.74496644])),
(38, nan, array([ 1.73152404, 1.80328159, 0.76510067])),
(39, nan, array([ 1.76747832, 1.76805554, 0.7852349 ])),
(40, nan, array([ 1.8027161 , 1.73211277, 0.80536913])),
(41, nan, array([ 1.83722311, 1.69546785, 0.82550336])),
(42, nan, array([ 1.87098536, 1.65813563, 0.84563758])),
(43, nan, array([ 1.90398916, 1.62013125, 0.86577181])),
(44, nan, array([ 1.93622114, 1.58147011, 0.88590604])),
(45, nan, array([ 1.96766822, 1.54216788, 0.90604027])),
(46, nan, array([ 1.99831766, 1.5022405 , 0.9261745 ])),
(47, nan, array([ 2.02815704, 1.46170415, 0.94630872])),
(48, nan, array([ 2.05717426, 1.42057526, 0.96644295])),
(49, nan, array([ 2.08535755, 1.37887052, 0.98657718])),
(50, nan, array([ 2.11269549, 1.33660681, 1.00671141])),
(51, nan, array([ 2.139177 , 1.29380128, 1.02684564])),
(52, nan, array([ 2.16479135, 1.25047127, 1.04697987])),
(53, nan, array([ 2.18952815, 1.20663436, 1.06711409])),
(54, nan, array([ 2.21337737, 1.16230831, 1.08724832])),
(55, nan, array([ 2.23632935, 1.11751109, 1.10738255])),
(56, nan, array([ 2.25837478, 1.07226086, 1.12751678])),
(57, nan, array([ 2.27950472, 1.02657597, 1.14765101])),
(58, nan, array([ 2.29971061, 0.98047493, 1.16778523])),
(59, nan, array([ 2.31898427, 0.93397643, 1.18791946])),
(60, nan, array([ 2.33731786, 0.88709932, 1.20805369])),
(61, nan, array([ 2.35470397, 0.83986261, 1.22818792])),
(62, nan, array([ 2.37113555, 0.79228543, 1.24832215])),
(63, nan, array([ 2.38660593, 0.74438709, 1.26845638])),
(64, nan, array([ 2.40110884, 0.69618699, 1.2885906 ])),
(65, nan, array([ 2.41463841, 0.64770468, 1.30872483])),
(66, nan, array([ 2.42718915, 0.5989598 , 1.32885906])),
(67, nan, array([ 2.43875597, 0.54997212, 1.34899329])),
(68, nan, array([ 2.44933418, 0.5007615 , 1.36912752])),
(69, nan, array([ 2.4589195 , 0.45134788, 1.38926174])),
(70, nan, array([ 2.46750803, 0.40175129, 1.40939597])),
(71, nan, array([ 2.47509631, 0.35199185, 1.4295302 ])),
(72, nan, array([ 2.48168124, 0.30208972, 1.44966443])),
(73, nan, array([ 2.48726017, 0.25206513, 1.46979866])),
(74, nan, array([ 2.49183083, 0.20193836, 1.48993289])),
(75, nan, array([ 2.49539137, 0.15172973, 1.51006711])),
(76, nan, array([ 2.49794034, 0.10145959, 1.53020134])),
(77, nan, array([ 2.49947672, 0.05114832, 1.55033557])),
(78, nan, array([ 2.49999987e+00, 8.16320328e-04, 1.57046980e+00])),
(79, nan, array([ 2.49950958, -0.04951601, 1.59060403])),
(80, nan, array([ 2.49800607, -0.09982827, 1.61073826])),
(81, nan, array([ 2.49548993, -0.15010006, 1.63087248])),
(82, nan, array([ 2.49196218, -0.20031101, 1.65100671])),
(83, nan, array([ 2.48742426, -0.25044075, 1.67114094])),
(84, nan, array([ 2.481878 , -0.30046898, 1.69127517])),
(85, nan, array([ 2.47532565, -0.3503754 , 1.7114094 ])),
(86, nan, array([ 2.46776987, -0.40013979, 1.73154362])),
(87, nan, array([ 2.45921373, -0.44974197, 1.75167785])),
(88, nan, array([ 2.44966068, -0.49916184, 1.77181208])),
(89, nan, array([ 2.43911461, -0.54837936, 1.79194631])),
(90, nan, array([ 2.42757978, -0.59737458, 1.81208054])),
(91, nan, array([ 2.41506088, -0.64612764, 1.83221477])),
(92, nan, array([ 2.40156298, -0.69461878, 1.85234899])),
(93, nan, array([ 2.38709155, -0.74282834, 1.87248322])),
(94, nan, array([ 2.37165245, -0.79073678, 1.89261745])),
(95, nan, array([ 2.35525195, -0.83832467, 1.91275168])),
(96, nan, array([ 2.33789669, -0.88557273, 1.93288591])),
(97, nan, array([ 2.31959371, -0.9324618 , 1.95302013])),
(98, nan, array([ 2.30035043, -0.97897288, 1.97315436])),
(99, nan, array([ 2.28017465, -1.02508711, 1.99328859])),
(100, nan, array([ 2.25907454, -1.07078579, 2.01342282])),
(101, nan, array([ 2.23705867, -1.11605041, 2.03355705])),
(102, nan, array([ 2.21413595, -1.1608626 , 2.05369128])),
(103, nan, array([ 2.19031568, -1.20520422, 2.0738255 ])),
(104, nan, array([ 2.16560752, -1.24905728, 2.09395973])),
(105, nan, array([ 2.14002147, -1.292404 , 2.11409396])),
(106, nan, array([ 2.11356792, -1.33522682, 2.13422819])),
(107, nan, array([ 2.08625758, -1.37750837, 2.15436242])),
(108, nan, array([ 2.05810153, -1.41923151, 2.17449664])),
(109, nan, array([ 2.02911118, -1.46037934, 2.19463087])),
(110, nan, array([ 1.99929828, -1.50093517, 2.2147651 ])),
(111, nan, array([ 1.96867492, -1.54088255, 2.23489933])),
(112, nan, array([ 1.93725351, -1.58020531, 2.25503356])),
(113, nan, array([ 1.90504679, -1.61888749, 2.27516779])),
(114, nan, array([ 1.87206782, -1.65691342, 2.29530201])),
(115, nan, array([ 1.83832996, -1.69426768, 2.31543624])),
(116, nan, array([ 1.80384688, -1.73093513, 2.33557047])),
(117, nan, array([ 1.76863258, -1.7669009 , 2.3557047 ])),
(118, nan, array([ 1.73270132, -1.80215042, 2.37583893])),
(119, nan, array([ 1.69606767, -1.8366694 , 2.39597315])),
(120, nan, array([ 1.65874647, -1.87044383, 2.41610738])),
(121, nan, array([ 1.62075287, -1.90346004, 2.43624161])),
(122, nan, array([ 1.58210226, -1.93570464, 2.45637584])),
(123, nan, array([ 1.5428103 , -1.96716455, 2.47651007])),
(124, nan, array([ 1.50289293, -1.99782703, 2.4966443 ])),
(125, nan, array([ 1.46236632, -2.02767964, 2.51677852])),
(126, nan, array([ 1.42124691, -2.05671029, 2.53691275])),
(127, nan, array([ 1.37955137, -2.0849072 , 2.55704698])),
(128, nan, array([ 1.33729659, -2.11225894, 2.57718121])),
(129, nan, array([ 1.29449971, -2.13875443, 2.59731544])),
(130, nan, array([ 1.25117807, -2.16438292, 2.61744966])),
(131, nan, array([ 1.20734924, -2.18913403, 2.63758389])),
(132, nan, array([ 1.16303098, -2.21299773, 2.65771812])),
(133, nan, array([ 1.11824126, -2.23596433, 2.67785235])),
(134, 0.0, array([ 1.07299823, -2.25802453, 2.69798658])),
(135, nan, array([ 1.02732024, -2.27916939, 2.71812081])),
(136, nan, array([ 0.9812258 , -2.29939034, 2.73825503])),
(137, nan, array([ 0.93473359, -2.31867917, 2.75838926])),
(138, nan, array([ 0.88786248, -2.33702808, 2.77852349])),
(139, nan, array([ 0.84063144, -2.35442961, 2.79865772])),
(140, nan, array([ 0.79305963, -2.37087672, 2.81879195])),
(141, nan, array([ 0.74516634, -2.38636274, 2.83892617])),
(142, nan, array([ 0.69697098, -2.40088139, 2.8590604 ])),
(143, nan, array([ 0.64849309, -2.41442679, 2.87919463])),
(144, nan, array([ 0.59975231, -2.42699344, 2.89932886])),
(145, nan, array([ 0.55076841, -2.43857626, 2.91946309])),
(146, nan, array([ 0.50156125, -2.44917054, 2.93959732])),
(147, nan, array([ 0.45215076, -2.45877199, 2.95973154])),
(148, nan, array([ 0.40255698, -2.46737672, 2.97986577])),
(149, nan, array([ 0.35280002, -2.47498124, 3. ])),
(150, nan, array([-0. , 2.5, -0. ])),
(151, nan, array([-0.05033217, 2.49949328, -0.02013423])),
(152, nan, array([-0.10064394, 2.49797334, -0.04026846])),
(153, nan, array([-0.1509149 , 2.49544078, -0.06040268])),
(154, nan, array([-0.20112469, 2.49189664, -0.08053691])),
(155, nan, array([-0.25125296, 2.48734235, -0.10067114])),
(156, nan, array([-0.30127936, 2.48177975, -0.12080537])),
(157, nan, array([-0.35118364, 2.47521111, -0.1409396 ])),
(158, nan, array([-0.40094556, 2.46763909, -0.16107383])),
(159, nan, array([-0.45054495, 2.45906674, -0.18120805])),
(160, nan, array([-0.49996169, 2.44949756, -0.20134228])),
(161, nan, array([-0.54917577, 2.43893542, -0.22147651])),
(162, nan, array([-0.59816722, 2.4273846 , -0.24161074])),
(163, nan, array([-0.64691619, 2.41484978, -0.26174497])),
(164, nan, array([-0.69540292, 2.40133604, -0.28187919])),
(165, nan, array([-0.74360776, 2.38684886, -0.30201342])),
(166, nan, array([-0.79151115, 2.37139413, -0.32214765])),
(167, nan, array([-0.83909369, 2.35497809, -0.34228188])),
(168, nan, array([-0.88633608, 2.3376074 , -0.36241611])),
(169, nan, array([-0.93321917, 2.31928911, -0.38255034])),
(170, nan, array([-0.97972396, 2.30003064, -0.40268456])),
(171, nan, array([-1.02583159, 2.27983981, -0.42281879])),
(172, nan, array([-1.07152338, 2.25872478, -0.44295302])),
(173, nan, array([-1.11678081, 2.23669413, -0.46308725])),
(174, nan, array([-1.16158552, 2.21375678, -0.48322148])),
(175, nan, array([-1.20591935, 2.18992203, -0.5033557 ])),
(176, nan, array([-1.24976434, 2.16519955, -0.52348993])),
(177, nan, array([-1.29310271, 2.13959935, -0.54362416])),
(178, nan, array([-1.33591688, 2.11313182, -0.56375839])),
(179, nan, array([-1.37818951, 2.08580768, -0.58389262])),
(180, nan, array([-1.41990346, 2.057638 , -0.60402685])),
(181, nan, array([-1.46104182, 2.02863422, -0.62416107])),
(182, nan, array([-1.50158791, 1.99880808, -0.6442953 ])),
(183, nan, array([-1.5415253 , 1.96817168, -0.66442953])),
(184, nan, array([-1.58083779, 1.93673743, -0.68456376])),
(185, nan, array([-1.61950946, 1.90451808, -0.70469799])),
(186, nan, array([-1.65752462, 1.87152669, -0.72483221])),
(187, nan, array([-1.69486786, 1.83777663, -0.74496644])),
(188, nan, array([-1.73152404, 1.80328159, -0.76510067])),
(189, nan, array([-1.76747832, 1.76805554, -0.7852349 ])),
(190, nan, array([-1.8027161 , 1.73211277, -0.80536913])),
(191, nan, array([-1.83722311, 1.69546785, -0.82550336])),
(192, nan, array([-1.87098536, 1.65813563, -0.84563758])),
(193, nan, array([-1.90398916, 1.62013125, -0.86577181])),
(194, nan, array([-1.93622114, 1.58147011, -0.88590604])),
(195, nan, array([-1.96766822, 1.54216788, -0.90604027])),
(196, nan, array([-1.99831766, 1.5022405 , -0.9261745 ])),
(197, nan, array([-2.02815704, 1.46170415, -0.94630872])),
(198, nan, array([-2.05717426, 1.42057526, -0.96644295])),
(199, nan, array([-2.08535755, 1.37887052, -0.98657718])),
(200, nan, array([-2.11269549, 1.33660681, -1.00671141])),
(201, nan, array([-2.139177 , 1.29380128, -1.02684564])),
(202, nan, array([-2.16479135, 1.25047127, -1.04697987])),
(203, nan, array([-2.18952815, 1.20663436, -1.06711409])),
(204, nan, array([-2.21337737, 1.16230831, -1.08724832])),
(205, nan, array([-2.23632935, 1.11751109, -1.10738255])),
(206, nan, array([-2.25837478, 1.07226086, -1.12751678])),
(207, nan, array([-2.27950472, 1.02657597, -1.14765101])),
(208, nan, array([-2.29971061, 0.98047493, -1.16778523])),
(209, nan, array([-2.31898427, 0.93397643, -1.18791946])),
(210, nan, array([-2.33731786, 0.88709932, -1.20805369])),
(211, nan, array([-2.35470397, 0.83986261, -1.22818792])),
(212, nan, array([-2.37113555, 0.79228543, -1.24832215])),
(213, nan, array([-2.38660593, 0.74438709, -1.26845638])),
(214, nan, array([-2.40110884, 0.69618699, -1.2885906 ])),
(215, nan, array([-2.41463841, 0.64770468, -1.30872483])),
(216, nan, array([-2.42718915, 0.5989598 , -1.32885906])),
(217, nan, array([-2.43875597, 0.54997212, -1.34899329])),
(218, nan, array([-2.44933418, 0.5007615 , -1.36912752])),
(219, nan, array([-2.4589195 , 0.45134788, -1.38926174])),
(220, nan, array([-2.46750803, 0.40175129, -1.40939597])),
(221, nan, array([-2.47509631, 0.35199185, -1.4295302 ])),
(222, nan, array([-2.48168124, 0.30208972, -1.44966443])),
(223, nan, array([-2.48726017, 0.25206513, -1.46979866])),
(224, nan, array([-2.49183083, 0.20193836, -1.48993289])),
(225, nan, array([-2.49539137, 0.15172973, -1.51006711])),
(226, nan, array([-2.49794034, 0.10145959, -1.53020134])),
(227, nan, array([-2.49947672, 0.05114832, -1.55033557])),
(228, nan, array([ -2.49999987e+00, 8.16320328e-04, -1.57046980e+00])),
(229, nan, array([-2.49950958, -0.04951601, -1.59060403])),
(230, nan, array([-2.49800607, -0.09982827, -1.61073826])),
(231, nan, array([-2.49548993, -0.15010006, -1.63087248])),
(232, nan, array([-2.49196218, -0.20031101, -1.65100671])),
(233, nan, array([-2.48742426, -0.25044075, -1.67114094])),
(234, nan, array([-2.481878 , -0.30046898, -1.69127517])),
(235, nan, array([-2.47532565, -0.3503754 , -1.7114094 ])),
(236, nan, array([-2.46776987, -0.40013979, -1.73154362])),
(237, nan, array([-2.45921373, -0.44974197, -1.75167785])),
(238, nan, array([-2.44966068, -0.49916184, -1.77181208])),
(239, nan, array([-2.43911461, -0.54837936, -1.79194631])),
(240, nan, array([-2.42757978, -0.59737458, -1.81208054])),
(241, nan, array([-2.41506088, -0.64612764, -1.83221477])),
(242, nan, array([-2.40156298, -0.69461878, -1.85234899])),
(243, nan, array([-2.38709155, -0.74282834, -1.87248322])),
(244, nan, array([-2.37165245, -0.79073678, -1.89261745])),
(245, nan, array([-2.35525195, -0.83832467, -1.91275168])),
(246, nan, array([-2.33789669, -0.88557273, -1.93288591])),
(247, nan, array([-2.31959371, -0.9324618 , -1.95302013])),
(248, nan, array([-2.30035043, -0.97897288, -1.97315436])),
(249, nan, array([-2.28017465, -1.02508711, -1.99328859])),
(250, nan, array([-2.25907454, -1.07078579, -2.01342282])),
(251, nan, array([-2.23705867, -1.11605041, -2.03355705])),
(252, nan, array([-2.21413595, -1.1608626 , -2.05369128])),
(253, nan, array([-2.19031568, -1.20520422, -2.0738255 ])),
(254, nan, array([-2.16560752, -1.24905728, -2.09395973])),
(255, nan, array([-2.14002147, -1.292404 , -2.11409396])),
(256, nan, array([-2.11356792, -1.33522682, -2.13422819])),
(257, nan, array([-2.08625758, -1.37750837, -2.15436242])),
(258, nan, array([-2.05810153, -1.41923151, -2.17449664])),
(259, nan, array([-2.02911118, -1.46037934, -2.19463087])),
(260, nan, array([-1.99929828, -1.50093517, -2.2147651 ])),
(261, nan, array([-1.96867492, -1.54088255, -2.23489933])),
(262, nan, array([-1.93725351, -1.58020531, -2.25503356])),
(263, nan, array([-1.90504679, -1.61888749, -2.27516779])),
(264, nan, array([-1.87206782, -1.65691342, -2.29530201])),
(265, nan, array([-1.83832996, -1.69426768, -2.31543624])),
(266, nan, array([-1.80384688, -1.73093513, -2.33557047])),
(267, nan, array([-1.76863258, -1.7669009 , -2.3557047 ])),
(268, nan, array([-1.73270132, -1.80215042, -2.37583893])),
(269, nan, array([-1.69606767, -1.8366694 , -2.39597315])),
(270, nan, array([-1.65874647, -1.87044383, -2.41610738])),
(271, nan, array([-1.62075287, -1.90346004, -2.43624161])),
(272, nan, array([-1.58210226, -1.93570464, -2.45637584])),
(273, nan, array([-1.5428103 , -1.96716455, -2.47651007])),
(274, nan, array([-1.50289293, -1.99782703, -2.4966443 ])),
(275, nan, array([-1.46236632, -2.02767964, -2.51677852])),
(276, nan, array([-1.42124691, -2.05671029, -2.53691275])),
(277, nan, array([-1.37955137, -2.0849072 , -2.55704698])),
(278, nan, array([-1.33729659, -2.11225894, -2.57718121])),
(279, nan, array([-1.29449971, -2.13875443, -2.59731544])),
(280, nan, array([-1.25117807, -2.16438292, -2.61744966])),
(281, nan, array([-1.20734924, -2.18913403, -2.63758389])),
(282, nan, array([-1.16303098, -2.21299773, -2.65771812])),
(283, 1.0, array([-1.11824126, -2.23596433, -2.67785235])),
(284, nan, array([-1.07299823, -2.25802453, -2.69798658])),
(285, nan, array([-1.02732024, -2.27916939, -2.71812081])),
(286, nan, array([-0.9812258 , -2.29939034, -2.73825503])),
(287, nan, array([-0.93473359, -2.31867917, -2.75838926])),
(288, nan, array([-0.88786248, -2.33702808, -2.77852349])),
(289, nan, array([-0.84063144, -2.35442961, -2.79865772])),
(290, nan, array([-0.79305963, -2.37087672, -2.81879195])),
(291, nan, array([-0.74516634, -2.38636274, -2.83892617])),
(292, nan, array([-0.69697098, -2.40088139, -2.8590604 ])),
(293, nan, array([-0.64849309, -2.41442679, -2.87919463])),
(294, nan, array([-0.59975231, -2.42699344, -2.89932886])),
(295, nan, array([-0.55076841, -2.43857626, -2.91946309])),
(296, nan, array([-0.50156125, -2.44917054, -2.93959732])),
(297, nan, array([-0.45215076, -2.45877199, -2.95973154])),
(298, nan, array([-0.40255698, -2.46737672, -2.97986577])),
(299, nan, array([-0.35280002, -2.47498124, -3. ]))]
In [5]:
def _compute_bfs(vec_1, vec_2, sigma=0.42):
return np.exp(-vec_1.squared_distance(vec_2)/sigma**2)
def _tolerence_cut(value, tol=10e-10):
if value <= tol:
return 0
else:
return value
def _scale_data_frame(df, vector=None):
if vector:
to_dense = lambda x: ml_linalg.DenseVector(x.toArray())
df = df.withColumn(vector, F.udf(to_dense, ml_linalg.VectorUDT())(vector) )
scaler = feature.StandardScaler(
withMean=True, withStd=True,
inputCol='vector', outputCol='std_vector')
model = scaler.fit(df)
return (model
.transform(df)
.select([i for i in df.columns if i != vector]+[scaler.getOutputCol()])
.withColumnRenamed(existing=scaler.getOutputCol(), new=vector))
def do_cartesian(sc, df, index=None, vector=None, **kwargs):
sigma = kwargs.get('sigma', 0.42)
tol = kwargs.get('tolerance', 10e-10)
scaled_df = _scale_data_frame(df, vector=vector)
bc_vec = sc.broadcast(scaled_df.select(index, vector).rdd.collectAsMap())
if index:
print(bc_vec.value[0])
index_rdd = df.rdd.map(lambda x: x[index]).cache()
cartesian_demon = index_rdd.cartesian(index_rdd).filter(lambda x: x[0] >= x[1])
cartesian_distance_demon = cartesian_demon.map(lambda x: MatrixEntry(x[0],x[1], _compute_bfs(
bc_vec.value.get(x[0]),
bc_vec.value.get(x[1]),
sigma)))
return cartesian_distance_demon.filter(lambda x: _tolerence_cut(x.value) )
def triangle_mat_summation(mat_element):
if mat_element.j == mat_element.i:
return (mat_element.i, mat_element.value),
else:
return (mat_element.i, mat_element.value), (mat_element.j, mat_element.value)
In [6]:
spring_data = generate_springs(n,a,1)
string_rdd = sc.parallelize(spring_data).map(lambda x: pyspark.Row(id=x[0],label=x[1], vector=ml_linalg.DenseVector(x[2])))
string_df = string_rdd.toDF()
# rdd = sc.range(10).map(lambda x: pyspark.Row(id=x, vector=ml_linalg.DenseVector(np.random.randint(0,9,size=10))))
# df = rdd.toDF()
In [7]:
string_df.show(300,False)
+---+-----+---------------------------------------------------------------+
|id |label|vector |
+---+-----+---------------------------------------------------------------+
|0 |NaN |[0.0,2.5,0.0] |
|1 |NaN |[0.05033216963986689,2.499493283187483,0.020134228187919462] |
|2 |NaN |[0.10064393595448022,2.4979733381594746,0.040268456375838924] |
|3 |NaN |[0.15091490388955273,2.495440781061335,0.060402684563758385] |
|4 |NaN |[0.20112469492937707,2.491896638524472,0.08053691275167785] |
|5 |NaN |[0.25125295535773495,2.487342347250174,0.10067114093959731] |
|6 |NaN |[0.3012793645087535,2.4817797534272055,0.12080536912751677] |
|7 |NaN |[0.35118364300436306,2.475211111983417,0.14093959731543623] |
|8 |NaN |[0.40094556097501977,2.467639085671652,0.1610738255033557] |
|9 |NaN |[0.4505449462603564,2.45906674399034,0.18120805369127516] |
|10 |NaN |[0.49996169258644135,2.449497561939203,0.20134228187919462] |
|11 |NaN |[0.5491757677163279,2.4389354186105834,0.22147651006711408] |
|12 |NaN |[0.5981672215705914,2.427384595616961,0.24161073825503354] |
|13 |NaN |[0.6469161943145619,2.4148497753553047,0.261744966442953] |
|14 |NaN |[0.6954029244089734,2.4013360391089473,0.28187919463087246] |
|15 |NaN |[0.7436077566207682,2.386848864987775,0.30201342281879195] |
|16 |NaN |[0.7915111499908044,2.3713941257075413,0.3221476510067114] |
|17 |NaN |[0.8390936857552446,2.354978086209228,0.3422818791946308] |
|18 |NaN |[0.8863360752174059,2.337607401119402,0.3624161073825503] |
|19 |NaN |[0.9332191675668848,2.319289112052607,0.3825503355704698] |
|20 |NaN |[0.9797239576427884,2.3000306447568803,0.40268456375838924] |
|21 |NaN |[1.0258315936379212,2.2798398061035523,0.42281879194630867] |
|22 |NaN |[1.0715233847408085,2.2587247809225457,0.44295302013422816] |
|23 |NaN |[1.1167808087124533,2.2366941286844653,0.46308724832214765] |
|24 |NaN |[1.1615855193947613,2.213756780030815,0.4832214765100671] |
|25 |NaN |[1.2059193541475863,2.1899220331537532,0.5033557046979865] |
|26 |NaN |[1.2497643412113826,2.165199550026851,0.523489932885906] |
|27 |NaN |[1.2931027069924783,2.139599352488387,0.5436241610738255] |
|28 |NaN |[1.3359168832680184,2.1131318181787577,0.5637583892617449] |
|29 |NaN |[1.3781895143076572,2.0858076763336606,0.5838926174496644] |
|30 |NaN |[1.4199034639091086,2.0576380034347426,0.6040268456375839] |
|31 |NaN |[1.4610418223447088,2.0286342187194943,0.6241610738255033] |
|32 |NaN |[1.5015879132161714,1.9988080795521876,0.6442953020134228] |
|33 |NaN |[1.5415253002147546,1.9681716766577582,0.6644295302013422] |
|34 |0.0 |[1.5808377937841067,1.936737429220543,0.6845637583892616] |
|35 |NaN |[1.6195094576830813,1.9045180798498742,0.7046979865771812] |
|36 |NaN |[1.6575246154458647,1.8715266894145641,0.7248322147651006] |
|37 |NaN |[1.6948678567368027,1.8377766317483735,0.7449664429530201] |
|38 |NaN |[1.7315240435973387,1.8032815882286162,0.7651006711409396] |
|39 |NaN |[1.76747831658254,1.7680555422300936,0.785234899328859] |
|40 |NaN |[1.8027161007847243,1.7321127734566013,0.8053691275167785] |
|41 |NaN |[1.8372231117417384,1.695467852152321,0.8255033557046979] |
|42 |NaN |[1.8709853612275014,1.658135633195426,0.8456375838926173] |
|43 |NaN |[1.9039891629224588,1.6201312500763128,0.8657718120805369] |
|44 |NaN |[1.9362211379616536,1.581470108762881,0.8859060402684563] |
|45 |NaN |[1.9676682203581641,1.542167881455367,0.9060402684563758] |
|46 |NaN |[1.9983176622997065,1.5022405002332473,0.9261744966442953] |
|47 |NaN |[2.028157039316259,1.4617041505967978,0.9463087248322147] |
|48 |NaN |[2.057174255316615,1.4205752649059222,0.9664429530201342] |
|49 |NaN |[2.085357547491814,1.378870515718908,0.9865771812080536] |
|50 |NaN |[2.1126954910834743,1.3366068090338146,1.006711409395973] |
|51 |NaN |[2.1391770040150866,1.2938012774352314,1.0268456375838926] |
|52 |NaN |[2.1647913513843915,1.2504712731491832,1.046979865771812] |
|53 |NaN |[2.1895281498150267,1.206634361008995,1.0671140939597314] |
|54 |NaN |[2.2133773716656697,1.1623083113349797,1.087248322147651] |
|55 |NaN |[2.2363293490949783,1.1175110927308198,1.1073825503355703] |
|56 |NaN |[2.258374777980678,1.0722608647995706,1.1275167785234899] |
|57 |NaN |[2.2795047216912043,1.0265759707822433,1.1476510067114094] |
|58 |NaN |[2.2997106147083763,0.9804749301219393,1.1677852348993287] |
|59 |NaN |[2.3189842660996316,0.9339764309565596,1.1879194630872483] |
|60 |NaN |[2.337317862838411,0.8870993225431317,1.2080536912751678] |
|61 |NaN |[2.354703972971353,0.8398626076168204,1.2281879194630871] |
|62 |NaN |[2.3711355486310115,0.7922854346877217,1.2483221476510067] |
|63 |NaN |[2.3866059288928714,0.7443870902785688,1.268456375838926] |
|64 |NaN |[2.4011088424755136,0.6961869911064834,1.2885906040268456] |
|65 |NaN |[2.4146384102828238,0.6477046762119587,1.308724832214765] |
|66 |NaN |[2.427189147787222,0.5989597990382487,1.3288590604026844] |
|67 |NaN |[2.4387559672529466,0.5499721194643825,1.348993288590604] |
|68 |NaN |[2.449334179798485,0.5007614957950384,1.3691275167785233] |
|69 |NaN |[2.458919497297322,0.4513478767105094,1.3892617449664428] |
|70 |NaN |[2.4675080341162343,0.401751293180042,1.4093959731543624] |
|71 |NaN |[2.475096308690421,0.3519918503418113,1.4295302013422817] |
|72 |NaN |[2.481681244934838,0.30208971935283035,1.4496644295302012] |
|73 |NaN |[2.4872601734911624,0.25206512921210183,1.4697986577181208] |
|74 |NaN |[2.4918308328098773,0.20193835856031697,1.4899328859060401] |
|75 |NaN |[2.4953913700670456,0.15172972745943192,1.5100671140939597] |
|76 |NaN |[2.4979403419153963,0.10145958915545703,1.5302013422818792] |
|77 |NaN |[2.4994767150694166,0.051148321827789785,1.5503355704697985] |
|78 |NaN |[2.4999998667242207,8.163203284402642E-4,1.570469798657718] |
|79 |NaN |[2.4995095848080178,-0.04951601208549656,1.5906040268456374] |
|80 |NaN |[2.498006068068079,-0.09982827202278394,1.610738255033557] |
|81 |NaN |[2.4954899259901735,-0.15010006422903255,1.6308724832214765] |
|82 |NaN |[2.491962178551495,-0.20031100985439682,1.6510067114093958] |
|83 |NaN |[2.4874242558071935,-0.2504407547146213,1.6711409395973154] |
|84 |NaN |[2.4818779973106686,-0.30046897754208257,1.6912751677852347] |
|85 |NaN |[2.475325651367861,-0.3503753982234961,1.7114093959731542] |
|86 |NaN |[2.467769874125852,-0.4001397860209319,1.7315436241610738] |
|87 |NaN |[2.459213728496129,-0.4497419677728203,1.751677852348993] |
|88 |NaN |[2.4496606829129646,-0.4991618360716172,1.7718120805369126] |
|89 |NaN |[2.4391146099274055,-0.5483793574148107,1.7919463087248322] |
|90 |NaN |[2.427579784637442,-0.5973745803259721,1.8120805369127515] |
|91 |NaN |[2.4150608829549967,-0.6461276434425571,1.832214765100671] |
|92 |NaN |[2.401562979710435,-0.6946187835671707,1.8523489932885906] |
|93 |NaN |[2.387091546595364,-0.7428283436790449,1.87248322147651] |
|94 |NaN |[2.371652449944551,-0.7907367809024746,1.8926174496644295] |
|95 |NaN |[2.3552519483578713,-0.8383246744289768,1.9127516778523488] |
|96 |NaN |[2.3378966901632348,-0.8855727333899749,1.9328859060402683] |
|97 |NaN |[2.3195937107215316,-0.9324618046768006,1.9530201342281879] |
|98 |NaN |[2.3003504295746833,-0.9789728807048582,1.9731543624161072] |
|99 |NaN |[2.2801746474379585,-1.0250871071187955,1.9932885906040267] |
|100|NaN |[2.2590745430377686,-1.070785790435555,2.013422818791946] |
|101|NaN |[2.2370586697962294,-1.1160504056222211,2.033557046979866] |
|102|NaN |[2.214135952363833,-1.1608626036055705,2.053691275167785] |
|103|NaN |[2.1903156830016273,-1.2052042187103043,2.0738255033557045] |
|104|NaN |[2.1656075178143848,-1.2490572760229288,2.093959731543624] |
|105|NaN |[2.14002147283627,-1.2924039986783085,2.1140939597315436] |
|106|NaN |[2.1135679199706097,-1.3352268150659317,2.134228187919463] |
|107|NaN |[2.086257582785392,-1.377508365952981,2.154362416107382] |
|108|NaN |[2.058101532166224,-1.4192315115213021,2.174496644295302] |
|109|NaN |[2.029111181828484,-1.4603793383154295,2.1946308724832213] |
|110|NaN |[1.9992982836905053,-1.5009351660988555,2.2147651006711406] |
|111|NaN |[1.9686749231096603,-1.540882554615754,2.2348993288590604] |
|112|NaN |[1.93725351398328,-1.5802053102554217,2.2550335570469797] |
|113|NaN |[1.9050467937163853,-1.6188874926167411,2.275167785234899] |
|114|NaN |[1.8720678180582841,-1.656913420969996,2.295302013422819] |
|115|NaN |[1.838329955810118,-1.6942676806134176,2.315436241610738] |
|116|NaN |[1.803846883405502,-1.7309351291219026,2.3355704697986575] |
|117|NaN |[1.7686325793664628,-1.7669009024853466,2.3557046979865772] |
|118|NaN |[1.7327013186369205,-1.8021504211341175,2.3758389261744965] |
|119|NaN |[1.6960676667959993,-1.8366693958492297,2.395973154362416] |
|120|NaN |[1.6587464741535318,-1.8704438335548133,2.4161073825503356] |
|121|NaN |[1.6207528697301399,-1.9034600429905315,2.436241610738255] |
|122|NaN |[1.5821022551243265,-1.9357046402616596,2.4563758389261743] |
|123|NaN |[1.5428102982690786,-1.967164554264558,2.4765100671140936] |
|124|NaN |[1.5028929270805045,-1.9978270319853504,2.4966442953020134] |
|125|NaN |[1.4623663230010795,-2.027679643669656,2.5167785234899327] |
|126|NaN |[1.4212469144401163,-2.056710287861285,2.536912751677852] |
|127|NaN |[1.3795513701141244,-2.084907196307846,2.557046979865772] |
|128|NaN |[1.3372965922897593,-2.112258938731281,2.577181208053691] |
|129|NaN |[1.2944997099320865,-2.1387544274614005,2.5973154362416104] |
|130|NaN |[1.2511780717609562,-2.1643829219305295,2.61744966442953] |
|131|NaN |[1.2073492392182956,-2.189134033027444,2.6375838926174495] |
|132|NaN |[1.163030979349162,-2.2129977273088484,2.657718120805369] |
|133|NaN |[1.118241257599456,-2.2359643310666626,2.6778523489932886] |
|134|NaN |[1.0729982305332106,-2.258024534249484,2.697986577181208] |
|135|NaN |[1.0273202384723954,-2.2791693942366416,2.7181208053691273] |
|136|NaN |[0.9812257980622421,-2.2993903394632924,2.7382550335570466] |
|137|NaN |[0.934733594765085,-2.318679172895108,2.7583892617449663] |
|138|NaN |[0.8878624752857756,-2.3370280753511317,2.7785234899328857] |
|139|NaN |[0.8406314399317218,-2.354429608673472,2.798657718120805] |
|140|NaN |[0.7930596349106722,-2.3708767187425313,2.8187919463087248] |
|141|NaN |[0.7451663445693534,-2.3863627383365564,2.838926174496644] |
|142|NaN |[0.6969709835761027,-2.4008813898343586,2.8590604026845634] |
|143|NaN |[0.6484930890506792,-2.4144267877600902,2.879194630872483] |
|144|NaN |[0.5997523126444384,-2.4269934411690626,2.8993288590604025] |
|145|NaN |[0.5507684125740674,-2.438576255873628,2.919463087248322] |
|146|NaN |[0.5015612456121317,-2.449170536508229,2.9395973154362416] |
|147|NaN |[0.4521507590376701,-2.4587719884327743,2.959731543624161] |
|148|NaN |[0.4025569825500913,-2.467376719473572,2.9798657718120802] |
|149|NaN |[0.35280002014966805,-2.4749812415011134,3.0] |
|150|NaN |[-0.0,2.5,-0.0] |
|151|NaN |[-0.05033216963986689,2.499493283187483,-0.020134228187919462] |
|152|NaN |[-0.10064393595448022,2.4979733381594746,-0.040268456375838924]|
|153|NaN |[-0.15091490388955273,2.495440781061335,-0.060402684563758385] |
|154|NaN |[-0.20112469492937707,2.491896638524472,-0.08053691275167785] |
|155|NaN |[-0.25125295535773495,2.487342347250174,-0.10067114093959731] |
|156|NaN |[-0.3012793645087535,2.4817797534272055,-0.12080536912751677] |
|157|NaN |[-0.35118364300436306,2.475211111983417,-0.14093959731543623] |
|158|NaN |[-0.40094556097501977,2.467639085671652,-0.1610738255033557] |
|159|NaN |[-0.4505449462603564,2.45906674399034,-0.18120805369127516] |
|160|NaN |[-0.49996169258644135,2.449497561939203,-0.20134228187919462] |
|161|NaN |[-0.5491757677163279,2.4389354186105834,-0.22147651006711408] |
|162|NaN |[-0.5981672215705914,2.427384595616961,-0.24161073825503354] |
|163|NaN |[-0.6469161943145619,2.4148497753553047,-0.261744966442953] |
|164|NaN |[-0.6954029244089734,2.4013360391089473,-0.28187919463087246] |
|165|NaN |[-0.7436077566207682,2.386848864987775,-0.30201342281879195] |
|166|NaN |[-0.7915111499908044,2.3713941257075413,-0.3221476510067114] |
|167|NaN |[-0.8390936857552446,2.354978086209228,-0.3422818791946308] |
|168|NaN |[-0.8863360752174059,2.337607401119402,-0.3624161073825503] |
|169|NaN |[-0.9332191675668848,2.319289112052607,-0.3825503355704698] |
|170|NaN |[-0.9797239576427884,2.3000306447568803,-0.40268456375838924] |
|171|NaN |[-1.0258315936379212,2.2798398061035523,-0.42281879194630867] |
|172|NaN |[-1.0715233847408085,2.2587247809225457,-0.44295302013422816] |
|173|NaN |[-1.1167808087124533,2.2366941286844653,-0.46308724832214765] |
|174|NaN |[-1.1615855193947613,2.213756780030815,-0.4832214765100671] |
|175|NaN |[-1.2059193541475863,2.1899220331537532,-0.5033557046979865] |
|176|NaN |[-1.2497643412113826,2.165199550026851,-0.523489932885906] |
|177|NaN |[-1.2931027069924783,2.139599352488387,-0.5436241610738255] |
|178|NaN |[-1.3359168832680184,2.1131318181787577,-0.5637583892617449] |
|179|NaN |[-1.3781895143076572,2.0858076763336606,-0.5838926174496644] |
|180|NaN |[-1.4199034639091086,2.0576380034347426,-0.6040268456375839] |
|181|NaN |[-1.4610418223447088,2.0286342187194943,-0.6241610738255033] |
|182|NaN |[-1.5015879132161714,1.9988080795521876,-0.6442953020134228] |
|183|NaN |[-1.5415253002147546,1.9681716766577582,-0.6644295302013422] |
|184|NaN |[-1.5808377937841067,1.936737429220543,-0.6845637583892616] |
|185|NaN |[-1.6195094576830813,1.9045180798498742,-0.7046979865771812] |
|186|NaN |[-1.6575246154458647,1.8715266894145641,-0.7248322147651006] |
|187|NaN |[-1.6948678567368027,1.8377766317483735,-0.7449664429530201] |
|188|NaN |[-1.7315240435973387,1.8032815882286162,-0.7651006711409396] |
|189|NaN |[-1.76747831658254,1.7680555422300936,-0.785234899328859] |
|190|NaN |[-1.8027161007847243,1.7321127734566013,-0.8053691275167785] |
|191|NaN |[-1.8372231117417384,1.695467852152321,-0.8255033557046979] |
|192|NaN |[-1.8709853612275014,1.658135633195426,-0.8456375838926173] |
|193|NaN |[-1.9039891629224588,1.6201312500763128,-0.8657718120805369] |
|194|NaN |[-1.9362211379616536,1.581470108762881,-0.8859060402684563] |
|195|NaN |[-1.9676682203581641,1.542167881455367,-0.9060402684563758] |
|196|NaN |[-1.9983176622997065,1.5022405002332473,-0.9261744966442953] |
|197|NaN |[-2.028157039316259,1.4617041505967978,-0.9463087248322147] |
|198|NaN |[-2.057174255316615,1.4205752649059222,-0.9664429530201342] |
|199|NaN |[-2.085357547491814,1.378870515718908,-0.9865771812080536] |
|200|NaN |[-2.1126954910834743,1.3366068090338146,-1.006711409395973] |
|201|NaN |[-2.1391770040150866,1.2938012774352314,-1.0268456375838926] |
|202|1.0 |[-2.1647913513843915,1.2504712731491832,-1.046979865771812] |
|203|NaN |[-2.1895281498150267,1.206634361008995,-1.0671140939597314] |
|204|NaN |[-2.2133773716656697,1.1623083113349797,-1.087248322147651] |
|205|NaN |[-2.2363293490949783,1.1175110927308198,-1.1073825503355703] |
|206|NaN |[-2.258374777980678,1.0722608647995706,-1.1275167785234899] |
|207|NaN |[-2.2795047216912043,1.0265759707822433,-1.1476510067114094] |
|208|NaN |[-2.2997106147083763,0.9804749301219393,-1.1677852348993287] |
|209|NaN |[-2.3189842660996316,0.9339764309565596,-1.1879194630872483] |
|210|NaN |[-2.337317862838411,0.8870993225431317,-1.2080536912751678] |
|211|NaN |[-2.354703972971353,0.8398626076168204,-1.2281879194630871] |
|212|NaN |[-2.3711355486310115,0.7922854346877217,-1.2483221476510067] |
|213|NaN |[-2.3866059288928714,0.7443870902785688,-1.268456375838926] |
|214|NaN |[-2.4011088424755136,0.6961869911064834,-1.2885906040268456] |
|215|NaN |[-2.4146384102828238,0.6477046762119587,-1.308724832214765] |
|216|NaN |[-2.427189147787222,0.5989597990382487,-1.3288590604026844] |
|217|NaN |[-2.4387559672529466,0.5499721194643825,-1.348993288590604] |
|218|NaN |[-2.449334179798485,0.5007614957950384,-1.3691275167785233] |
|219|NaN |[-2.458919497297322,0.4513478767105094,-1.3892617449664428] |
|220|NaN |[-2.4675080341162343,0.401751293180042,-1.4093959731543624] |
|221|NaN |[-2.475096308690421,0.3519918503418113,-1.4295302013422817] |
|222|NaN |[-2.481681244934838,0.30208971935283035,-1.4496644295302012] |
|223|NaN |[-2.4872601734911624,0.25206512921210183,-1.4697986577181208] |
|224|NaN |[-2.4918308328098773,0.20193835856031697,-1.4899328859060401] |
|225|NaN |[-2.4953913700670456,0.15172972745943192,-1.5100671140939597] |
|226|NaN |[-2.4979403419153963,0.10145958915545703,-1.5302013422818792] |
|227|NaN |[-2.4994767150694166,0.051148321827789785,-1.5503355704697985] |
|228|NaN |[-2.4999998667242207,8.163203284402642E-4,-1.570469798657718] |
|229|NaN |[-2.4995095848080178,-0.04951601208549656,-1.5906040268456374] |
|230|NaN |[-2.498006068068079,-0.09982827202278394,-1.610738255033557] |
|231|NaN |[-2.4954899259901735,-0.15010006422903255,-1.6308724832214765] |
|232|NaN |[-2.491962178551495,-0.20031100985439682,-1.6510067114093958] |
|233|NaN |[-2.4874242558071935,-0.2504407547146213,-1.6711409395973154] |
|234|NaN |[-2.4818779973106686,-0.30046897754208257,-1.6912751677852347] |
|235|NaN |[-2.475325651367861,-0.3503753982234961,-1.7114093959731542] |
|236|NaN |[-2.467769874125852,-0.4001397860209319,-1.7315436241610738] |
|237|NaN |[-2.459213728496129,-0.4497419677728203,-1.751677852348993] |
|238|NaN |[-2.4496606829129646,-0.4991618360716172,-1.7718120805369126] |
|239|NaN |[-2.4391146099274055,-0.5483793574148107,-1.7919463087248322] |
|240|NaN |[-2.427579784637442,-0.5973745803259721,-1.8120805369127515] |
|241|NaN |[-2.4150608829549967,-0.6461276434425571,-1.832214765100671] |
|242|NaN |[-2.401562979710435,-0.6946187835671707,-1.8523489932885906] |
|243|NaN |[-2.387091546595364,-0.7428283436790449,-1.87248322147651] |
|244|NaN |[-2.371652449944551,-0.7907367809024746,-1.8926174496644295] |
|245|NaN |[-2.3552519483578713,-0.8383246744289768,-1.9127516778523488] |
|246|NaN |[-2.3378966901632348,-0.8855727333899749,-1.9328859060402683] |
|247|NaN |[-2.3195937107215316,-0.9324618046768006,-1.9530201342281879] |
|248|NaN |[-2.3003504295746833,-0.9789728807048582,-1.9731543624161072] |
|249|NaN |[-2.2801746474379585,-1.0250871071187955,-1.9932885906040267] |
|250|NaN |[-2.2590745430377686,-1.070785790435555,-2.013422818791946] |
|251|NaN |[-2.2370586697962294,-1.1160504056222211,-2.033557046979866] |
|252|NaN |[-2.214135952363833,-1.1608626036055705,-2.053691275167785] |
|253|NaN |[-2.1903156830016273,-1.2052042187103043,-2.0738255033557045] |
|254|NaN |[-2.1656075178143848,-1.2490572760229288,-2.093959731543624] |
|255|NaN |[-2.14002147283627,-1.2924039986783085,-2.1140939597315436] |
|256|NaN |[-2.1135679199706097,-1.3352268150659317,-2.134228187919463] |
|257|NaN |[-2.086257582785392,-1.377508365952981,-2.154362416107382] |
|258|NaN |[-2.058101532166224,-1.4192315115213021,-2.174496644295302] |
|259|NaN |[-2.029111181828484,-1.4603793383154295,-2.1946308724832213] |
|260|NaN |[-1.9992982836905053,-1.5009351660988555,-2.2147651006711406] |
|261|NaN |[-1.9686749231096603,-1.540882554615754,-2.2348993288590604] |
|262|NaN |[-1.93725351398328,-1.5802053102554217,-2.2550335570469797] |
|263|NaN |[-1.9050467937163853,-1.6188874926167411,-2.275167785234899] |
|264|NaN |[-1.8720678180582841,-1.656913420969996,-2.295302013422819] |
|265|NaN |[-1.838329955810118,-1.6942676806134176,-2.315436241610738] |
|266|NaN |[-1.803846883405502,-1.7309351291219026,-2.3355704697986575] |
|267|NaN |[-1.7686325793664628,-1.7669009024853466,-2.3557046979865772] |
|268|NaN |[-1.7327013186369205,-1.8021504211341175,-2.3758389261744965] |
|269|NaN |[-1.6960676667959993,-1.8366693958492297,-2.395973154362416] |
|270|NaN |[-1.6587464741535318,-1.8704438335548133,-2.4161073825503356] |
|271|NaN |[-1.6207528697301399,-1.9034600429905315,-2.436241610738255] |
|272|NaN |[-1.5821022551243265,-1.9357046402616596,-2.4563758389261743] |
|273|NaN |[-1.5428102982690786,-1.967164554264558,-2.4765100671140936] |
|274|NaN |[-1.5028929270805045,-1.9978270319853504,-2.4966442953020134] |
|275|NaN |[-1.4623663230010795,-2.027679643669656,-2.5167785234899327] |
|276|NaN |[-1.4212469144401163,-2.056710287861285,-2.536912751677852] |
|277|NaN |[-1.3795513701141244,-2.084907196307846,-2.557046979865772] |
|278|NaN |[-1.3372965922897593,-2.112258938731281,-2.577181208053691] |
|279|NaN |[-1.2944997099320865,-2.1387544274614005,-2.5973154362416104] |
|280|NaN |[-1.2511780717609562,-2.1643829219305295,-2.61744966442953] |
|281|NaN |[-1.2073492392182956,-2.189134033027444,-2.6375838926174495] |
|282|NaN |[-1.163030979349162,-2.2129977273088484,-2.657718120805369] |
|283|NaN |[-1.118241257599456,-2.2359643310666626,-2.6778523489932886] |
|284|NaN |[-1.0729982305332106,-2.258024534249484,-2.697986577181208] |
|285|NaN |[-1.0273202384723954,-2.2791693942366416,-2.7181208053691273] |
|286|NaN |[-0.9812257980622421,-2.2993903394632924,-2.7382550335570466] |
|287|NaN |[-0.934733594765085,-2.318679172895108,-2.7583892617449663] |
|288|NaN |[-0.8878624752857756,-2.3370280753511317,-2.7785234899328857] |
|289|NaN |[-0.8406314399317218,-2.354429608673472,-2.798657718120805] |
|290|NaN |[-0.7930596349106722,-2.3708767187425313,-2.8187919463087248] |
|291|NaN |[-0.7451663445693534,-2.3863627383365564,-2.838926174496644] |
|292|NaN |[-0.6969709835761027,-2.4008813898343586,-2.8590604026845634] |
|293|NaN |[-0.6484930890506792,-2.4144267877600902,-2.879194630872483] |
|294|NaN |[-0.5997523126444384,-2.4269934411690626,-2.8993288590604025] |
|295|NaN |[-0.5507684125740674,-2.438576255873628,-2.919463087248322] |
|296|NaN |[-0.5015612456121317,-2.449170536508229,-2.9395973154362416] |
|297|NaN |[-0.4521507590376701,-2.4587719884327743,-2.959731543624161] |
|298|NaN |[-0.4025569825500913,-2.467376719473572,-2.9798657718120802] |
|299|NaN |[-0.35280002014966805,-2.4749812415011134,-3.0] |
+---+-----+---------------------------------------------------------------+
In [8]:
demon = do_cartesian(sc, string_df,'id', 'vector')
demon.take(10)
[-6.10797542368e-17,1.37652733992,3.17294190766e-17]
Out[8]:
[MatrixEntry(0, 0, 1.0),
MatrixEntry(1, 0, 0.9948465948193068),
MatrixEntry(2, 0, 0.979546467464139),
MatrixEntry(1, 1, 1.0),
MatrixEntry(2, 1, 0.9948462833543696),
MatrixEntry(2, 2, 1.0),
MatrixEntry(3, 0, 0.9545715578949986),
MatrixEntry(4, 0, 0.9206812925488386),
MatrixEntry(5, 0, 0.8788839464495921),
MatrixEntry(6, 0, 0.8303864948563296)]
In [9]:
demon_matrix = CoordinateMatrix(demon, 300, 300)
row_summed_matrix = demon_matrix.entries.flatMap(triangle_mat_summation).reduceByKey(lambda x,y: x+y).collectAsMap()
bc_row_summed = sc.broadcast(row_summed_matrix)
In [10]:
transition_rdd = demon.map(lambda x: MatrixEntry(x.i, x.j, x.value/bc_row_summed.value.get(x.j) ))
col_summed_matrix = transition_rdd.flatMap(triangle_mat_summation).reduceByKey(lambda x,y: x+y).collectAsMap()
bc_col_summed = sc.broadcast(col_summed_matrix)
In [11]:
hat_transition_rdd = transition_rdd.map(lambda x: MatrixEntry(x.i,x.j, x.value/bc_col_summed.value.get(x.i)))
hat_transition_rdd.take(25)
Out[11]:
[MatrixEntry(0, 0, 0.03885626426621346),
MatrixEntry(1, 0, 0.03865634180049757),
MatrixEntry(2, 0, 0.03806368333210306),
MatrixEntry(1, 1, 0.038864896355216096),
MatrixEntry(2, 1, 0.03866647926373562),
MatrixEntry(2, 2, 0.03889149963010969),
MatrixEntry(3, 0, 0.0370975477380816),
MatrixEntry(4, 0, 0.035787897381467496),
MatrixEntry(5, 0, 0.03417381210218183),
MatrixEntry(6, 0, 0.03230156446721945),
MatrixEntry(3, 1, 0.03807621921677726),
MatrixEntry(3, 2, 0.0386955784355721),
MatrixEntry(4, 1, 0.037112976697733155),
MatrixEntry(4, 2, 0.03810821923499331),
MatrixEntry(5, 1, 0.0358062355961513),
MatrixEntry(5, 2, 0.0371478086029921),
MatrixEntry(6, 1, 0.034194599635376646),
MatrixEntry(6, 2, 0.03584334262962627),
MatrixEntry(3, 3, 0.03893652382769003),
MatrixEntry(4, 3, 0.03874383722620631),
MatrixEntry(4, 4, 0.038999766747288854),
MatrixEntry(5, 3, 0.03815955761840735),
MatrixEntry(5, 4, 0.038810729125191366),
MatrixEntry(5, 5, 0.03908034393490406),
MatrixEntry(6, 3, 0.03720156865894708)]
In [12]:
def generate_label_matrix(df):
Y_L = df.filter(~F.isnan(F.col('label'))).select('id','label').cache()
Y_U = df.filter(F.isnan(F.col('label'))).select('id').cache()
Y_max = Y_L.groupby().max('label').collect()[0][0]
Y_L_rdd = Y_L.rdd.map(lambda x: MatrixEntry(i=x['id'], j=x['label'], value=1.0))
Y_L_mat = CoordinateMatrix(Y_L_rdd, numRows=df.count(), numCols=Y_max+1)
Y_U_rdd = Y_U.rdd.flatMap(lambda x: [MatrixEntry(i=x['id'], j=idx, value=.50) for idx in range(int(Y_max+1))])
#print(Y_U_rdd.take(5))
Y_U_mat = CoordinateMatrix(Y_U_rdd, numRows=df.count(), numCols=Y_max+1)
return Y_L_mat, Y_U_mat
In [23]:
y_l, y_u = generate_label_matrix(string_df)
print(y_l.entries.take(5))
print(y_u.entries.take(5))
print(y_l.numRows(),y_l.numCols())
print(y_u.numRows(),y_u.numCols())
[MatrixEntry(34, 0, 1.0), MatrixEntry(202, 1, 1.0)]
[MatrixEntry(0, 0, 0.5), MatrixEntry(0, 1, 0.5), MatrixEntry(1, 0, 0.5), MatrixEntry(1, 1, 0.5), MatrixEntry(2, 0, 0.5)]
300 2
300 2
In [14]:
mat_hatter_rdd = CoordinateMatrix(hat_transition_rdd,numCols=2*n, numRows=2*n)
In [15]:
def naive_multiplication(A:CoordinateMatrix, B:CoordinateMatrix, is_triangle=False):
"""
A is the left matrix
B is the right matix
"""
if is_triangle:
left_rdd = (A.entries
.flatMap(lambda x: [((x.j, x.i), x.value),((x.i, x.j), x.value)])
.aggregateByKey(
zeroValue=(0.0,0.0),
seqFunc=lambda x,y: (x[0]+y, x[1] + 1 ),
combFunc=lambda a,b: (x[0] + y[0], x[1] + y[1]))
.mapValues(lambda x: x[0]/x[1])
.map(lambda x: (x[0][0], (x[0][1], x[1])))
)
else:
left_rdd = A.entries.map(lambda x: (x.j, (x.i, x.value)))
right_rdd = B.entries.map(lambda x: (x.i, (x.j, x.value)))
combined_rdd = (left_rdd
.join(right_rdd)
.map(lambda x: x[1])
.map(lambda x: ((x[0][0], x[1][0]), x[0][1]*x[1][1]))
.reduceByKey(lambda x,y: x+y)
.map(lambda x: MatrixEntry(i=x[0][0], j=x[0][1], value=x[1]))
)
return combined_rdd
In [17]:
new_y_l = naive_multiplication(mat_hatter_rdd, y_l, is_triangle=True)
n
Out[17]:
268
In [24]:
print(mat_hatter_rdd.numCols(),mat_hatter_rdd.numRows())
print(y_l.numCols(), y_l.numRows())
300 300
2 300
In [30]:
d = (mat_hatter_rdd.toBlockMatrix()).multiply(y_l.toBlockMatrix())
dd = d.toLocalMatrix()
dd.toArray()
Out[30]:
array([[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 4.11414063e-02, 0.00000000e+00],
[ 4.09159705e-02, 0.00000000e+00],
[ 4.02610398e-02, 0.00000000e+00],
[ 3.91964853e-02, 0.00000000e+00],
[ 3.77548143e-02, 0.00000000e+00],
[ 3.59795079e-02, 0.00000000e+00],
[ 3.39228207e-02, 0.00000000e+00],
[ 3.16431987e-02, 0.00000000e+00],
[ 2.92024973e-02, 0.00000000e+00],
[ 2.66631847e-02, 0.00000000e+00],
[ 2.40857082e-02, 0.00000000e+00],
[ 2.15261770e-02, 0.00000000e+00],
[ 1.90344779e-02, 0.00000000e+00],
[ 1.66529001e-02, 0.00000000e+00],
[ 1.44153004e-02, 0.00000000e+00],
[ 1.23467964e-02, 0.00000000e+00],
[ 1.04639419e-02, 0.00000000e+00],
[ 8.77530594e-03, 0.00000000e+00],
[ 7.28236444e-03, 0.00000000e+00],
[ 5.98060118e-03, 0.00000000e+00],
[ 4.86071852e-03, 0.00000000e+00],
[ 3.90986730e-03, 0.00000000e+00],
[ 3.11282001e-03, 0.00000000e+00],
[ 2.45302961e-03, 0.00000000e+00],
[ 1.91353544e-03, 0.00000000e+00],
[ 1.47769610e-03, 0.00000000e+00],
[ 1.12974496e-03, 0.00000000e+00],
[ 8.55176827e-04, 0.00000000e+00],
[ 6.40983766e-04, 0.00000000e+00],
[ 4.75763241e-04, 0.00000000e+00],
[ 3.49724797e-04, 0.00000000e+00],
[ 2.54621091e-04, 0.00000000e+00],
[ 1.83627287e-04, 0.00000000e+00],
[ 1.31189539e-04, 0.00000000e+00],
[ 9.28594983e-05, 0.00000000e+00],
[ 6.51277897e-05, 0.00000000e+00],
[ 4.52656422e-05, 0.00000000e+00],
[ 3.11806056e-05, 0.00000000e+00],
[ 2.12895648e-05, 0.00000000e+00],
[ 1.44101981e-05, 0.00000000e+00],
[ 9.67052940e-06, 0.00000000e+00],
[ 6.43525159e-06, 0.00000000e+00],
[ 4.24693156e-06, 0.00000000e+00],
[ 2.77997258e-06, 0.00000000e+00],
[ 1.80519418e-06, 0.00000000e+00],
[ 1.16302673e-06, 0.00000000e+00],
[ 7.43538640e-07, 0.00000000e+00],
[ 4.71772570e-07, 0.00000000e+00],
[ 2.97129529e-07, 0.00000000e+00],
[ 1.85786032e-07, 0.00000000e+00],
[ 1.15346908e-07, 0.00000000e+00],
[ 7.11209298e-08, 0.00000000e+00],
[ 4.35573781e-08, 0.00000000e+00],
[ 2.65017431e-08, 0.00000000e+00],
[ 1.60218406e-08, 0.00000000e+00],
[ 9.62617408e-09, 0.00000000e+00],
[ 5.74880586e-09, 0.00000000e+00],
[ 3.41322288e-09, 0.00000000e+00],
[ 2.01509858e-09, 0.00000000e+00],
[ 1.18319204e-09, 0.00000000e+00],
[ 6.91074964e-10, 0.00000000e+00],
[ 4.01597266e-10, 0.00000000e+00],
[ 2.32240327e-10, 0.00000000e+00],
[ 1.33675669e-10, 0.00000000e+00],
[ 7.65988670e-11, 0.00000000e+00],
[ 4.37054698e-11, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 1.20220526e-04, 0.00000000e+00],
[ 8.62894868e-05, 0.00000000e+00],
[ 6.14453840e-05, 0.00000000e+00],
[ 4.34122828e-05, 0.00000000e+00],
[ 3.04345387e-05, 0.00000000e+00],
[ 2.11733571e-05, 0.00000000e+00],
[ 1.46190717e-05, 0.00000000e+00],
[ 1.00183403e-05, 0.00000000e+00],
[ 6.81487343e-06, 0.00000000e+00],
[ 4.60201277e-06, 0.00000000e+00],
[ 3.08539079e-06, 0.00000000e+00],
[ 2.05395956e-06, 0.00000000e+00],
[ 1.35782207e-06, 0.00000000e+00],
[ 8.91489020e-07, 0.00000000e+00],
[ 5.81389355e-07, 0.00000000e+00],
[ 3.76664412e-07, 0.00000000e+00],
[ 2.42459640e-07, 0.00000000e+00],
[ 1.55090310e-07, 0.00000000e+00],
[ 9.85949979e-08, 0.00000000e+00],
[ 6.23040201e-08, 0.00000000e+00],
[ 3.91412862e-08, 0.00000000e+00],
[ 2.44500288e-08, 0.00000000e+00],
[ 1.51885598e-08, 0.00000000e+00],
[ 9.38457151e-09, 0.00000000e+00],
[ 5.76820182e-09, 0.00000000e+00],
[ 3.52745756e-09, 0.00000000e+00],
[ 2.14657128e-09, 0.00000000e+00],
[ 1.30004202e-09, 0.00000000e+00],
[ 7.83725543e-10, 0.00000000e+00],
[ 4.70360383e-10, 0.00000000e+00],
[ 2.81075413e-10, 0.00000000e+00],
[ 1.67265114e-10, 0.00000000e+00],
[ 9.91386735e-11, 0.00000000e+00],
[ 5.85329994e-11, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 4.15885008e-02],
[ 0.00000000e+00, 4.13603478e-02],
[ 0.00000000e+00, 4.06861793e-02],
[ 0.00000000e+00, 3.95872540e-02],
[ 0.00000000e+00, 3.80982860e-02],
[ 0.00000000e+00, 3.62656001e-02],
[ 0.00000000e+00, 3.41447067e-02],
[ 0.00000000e+00, 3.17974768e-02],
[ 0.00000000e+00, 2.92891188e-02],
[ 0.00000000e+00, 2.66851664e-02],
[ 0.00000000e+00, 2.40486690e-02],
[ 0.00000000e+00, 2.14377513e-02],
[ 0.00000000e+00, 1.89036638e-02],
[ 0.00000000e+00, 1.64893987e-02],
[ 0.00000000e+00, 1.42288965e-02],
[ 0.00000000e+00, 1.21468223e-02],
[ 0.00000000e+00, 1.02588495e-02],
[ 0.00000000e+00, 8.57236455e-03],
[ 0.00000000e+00, 7.08748324e-03],
[ 0.00000000e+00, 5.79826864e-03],
[ 0.00000000e+00, 4.69404133e-03],
[ 0.00000000e+00, 3.76068674e-03],
[ 0.00000000e+00, 2.98188184e-03],
[ 0.00000000e+00, 2.34018412e-03],
[ 0.00000000e+00, 1.81794694e-03],
[ 0.00000000e+00, 1.39804495e-03],
[ 0.00000000e+00, 1.06440995e-03],
[ 0.00000000e+00, 8.02390128e-04],
[ 0.00000000e+00, 5.98954708e-04],
[ 0.00000000e+00, 4.42770732e-04],
[ 0.00000000e+00, 3.24180552e-04],
[ 0.00000000e+00, 2.35107552e-04],
[ 0.00000000e+00, 1.68914926e-04],
[ 0.00000000e+00, 1.20238440e-04],
[ 0.00000000e+00, 8.48098497e-05],
[ 0.00000000e+00, 5.92833096e-05],
[ 0.00000000e+00, 4.10732445e-05],
[ 0.00000000e+00, 2.82088199e-05],
[ 0.00000000e+00, 1.92074948e-05],
[ 0.00000000e+00, 1.29681675e-05],
[ 0.00000000e+00, 8.68305082e-06],
[ 0.00000000e+00, 5.76656552e-06],
[ 0.00000000e+00, 3.79909457e-06],
[ 0.00000000e+00, 2.48330049e-06],
[ 0.00000000e+00, 1.61077354e-06],
[ 0.00000000e+00, 1.03697169e-06],
[ 0.00000000e+00, 6.62672630e-07],
[ 0.00000000e+00, 4.20440677e-07],
[ 0.00000000e+00, 2.64886063e-07],
[ 0.00000000e+00, 1.65744279e-07],
[ 0.00000000e+00, 1.03019778e-07],
[ 0.00000000e+00, 6.36185874e-08],
[ 0.00000000e+00, 3.90398932e-08],
[ 0.00000000e+00, 2.38108729e-08],
[ 0.00000000e+00, 1.44366376e-08],
[ 0.00000000e+00, 8.70289535e-09],
[ 0.00000000e+00, 5.21737826e-09],
[ 0.00000000e+00, 3.11111844e-09],
[ 0.00000000e+00, 1.84561925e-09],
[ 0.00000000e+00, 1.08947076e-09],
[ 0.00000000e+00, 6.40065386e-10],
[ 0.00000000e+00, 3.74331726e-10],
[ 0.00000000e+00, 2.17972418e-10],
[ 0.00000000e+00, 1.26400557e-10],
[ 0.00000000e+00, 7.30112532e-11],
[ 0.00000000e+00, 4.20161013e-11],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00],
[ 0.00000000e+00, 0.00000000e+00]])
In [ ]:
Content source: mssalvador/WorkflowCleaning
Similar notebooks: